| 1 | | [ Aug 22 02:15:30 Enabled. ] |
| 2 | | [ Aug 22 02:15:30 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] |
| 3 | | [ Aug 22 02:15:30 Method "start" exited with status 0. ] |
| 4 | | [ Aug 22 02:15:30 Rereading configuration. ] |
| 5 | | [ Aug 22 02:15:30 No 'refresh' method defined. Treating as :true. ] |
| 6 | | note: configured to log to "/dev/stdout" |
| 7 | 2024-08-22T02:15:30.412Z | INFO | SledAgent: Ensuring contract reaper thread
file = illumos-utils/src/running_zone.rs:113
|
| 8 | 2024-08-22T02:15:30.412Z | INFO | SledAgent (BootstrapAgentStartup): finding links ["net0", "net1"]
file = sled-agent/src/bootstrap/pre_server.rs:255
|
| 9 | 2024-08-22T02:15:30.466Z | INFO | SledAgent (BootstrapAgentStartup): Starting mg-ddm service
addr-objs = [AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]
file = sled-agent/src/bootstrap/pre_server.rs:262
|
| 10 | 2024-08-22T02:15:30.466Z | INFO | SledAgent (BootstrapAgentStartup): Importing mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:42
path = /opt/oxide/mg-ddm/pkg/ddm/manifest.xml
|
| 11 | 2024-08-22T02:15:30.581Z | INFO | SledAgent (BootstrapAgentStartup): Setting mg-ddm interfaces
file = sled-agent/src/bootstrap/maghemite.rs:50
interfaces = ("net0/ll" "net1/ll")
|
| 12 | 2024-08-22T02:15:30.590Z | INFO | SledAgent (BootstrapAgentStartup): Enabling mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:56
|
| 13 | 2024-08-22T02:15:30.598Z | INFO | SledAgent (BootstrapAgentStartup): Ensuring zfs key directory exists
file = sled-agent/src/bootstrap/pre_server.rs:272
path = /var/run/oxide/
|
| 14 | 2024-08-22T02:15:31.202Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }
|
| 15 | 2024-08-22T02:15:31.340Z | INFO | SledAgent: Starting KeyManager
file = sled-agent/src/long_running_tasks.rs:126
|
| 16 | 2024-08-22T02:15:31.340Z | INFO | SledAgent: Starting StorageManager
file = sled-agent/src/long_running_tasks.rs:138
|
| 17 | 2024-08-22T02:15:31.340Z | INFO | SledAgent: Starting StorageMonitor
file = sled-agent/src/long_running_tasks.rs:151
|
| 18 | 2024-08-22T02:15:31.340Z | INFO | SledAgent: Starting HardwareManager
file = sled-agent/src/long_running_tasks.rs:173
nongimlet_observed_disks = []
sled_mode = Scrimlet { asic: SoftNpuZone }
|
| 19 | 2024-08-22T02:15:31.340Z | INFO | SledAgent (DumpSetup-worker): DumpSetup poll loop started.
file = sled-agent/src/dump_setup.rs:520
|
| 20 | 2024-08-22T02:15:31.340Z | INFO | SledAgent (HardwareManager): Creating HardwareManager
file = sled-hardware/src/illumos/mod.rs:716
|
| 21 | 2024-08-22T02:15:31.343Z | INFO | SledAgent (HardwareManager): Generated i86pc baseboard Pc { identifier: "sock", model: "i86pc" }
file = sled-hardware/src/illumos/mod.rs:623
|
| 22 | 2024-08-22T02:15:31.343Z | WARN | SledAgent (HardwareManager): Device is not a Gimlet (i86pc), proceeding with null hardware view
file = sled-hardware/src/illumos/mod.rs:765
|
| 23 | 2024-08-22T02:15:31.343Z | INFO | SledAgent: Starting HardwareMonitor
file = sled-agent/src/long_running_tasks.rs:187
|
| 24 | 2024-08-22T02:15:31.343Z | INFO | SledAgent (HardwareManager): Monitoring for hardware updates
file = sled-hardware/src/illumos/mod.rs:824
|
| 25 | 2024-08-22T02:15:31.343Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/m2_0.vdev
|
| 26 | 2024-08-22T02:15:31.344Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
disks = {}
file = sled-agent/src/hardware_monitor.rs:271
underlay_network_info = None
|
| 27 | 2024-08-22T02:15:31.344Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }), tx: ... }
file = sled-storage/src/manager.rs:406
|
| 28 | 2024-08-22T02:15:31.344Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 29 | 2024-08-22T02:15:31.344Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
path = /scratch/m2_0.vdev
|
| 30 | 2024-08-22T02:15:31.355Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev
file = sled-hardware/src/disk.rs:360
|
| 31 | 2024-08-22T02:15:31.355Z | INFO | SledAgent (StorageResources): Formatting zpool with generated ID
file = sled-hardware/src/disk.rs:371
id = 6f31f2f8-a517-49b4-86c9-0d60b67db277 (zpool)
|
| 32 | 2024-08-22T02:15:31.485Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:307
zpool = ZpoolName { id: 6f31f2f8-a517-49b4-86c9-0d60b67db277 (zpool), kind: Internal }
|
| 33 | 2024-08-22T02:15:31.738Z | INFO | SledAgent (StorageManager): Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }
file = sled-storage/src/manager.rs:406
|
| 34 | 2024-08-22T02:15:31.738Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/m2_1.vdev
|
| 35 | 2024-08-22T02:15:31.738Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 36 | 2024-08-22T02:15:31.738Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 37 | 2024-08-22T02:15:31.738Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }), tx: ... }
file = sled-storage/src/manager.rs:406
|
| 38 | 2024-08-22T02:15:31.738Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 39 | 2024-08-22T02:15:31.738Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
path = /scratch/m2_1.vdev
|
| 40 | 2024-08-22T02:15:31.747Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev
file = sled-hardware/src/disk.rs:360
|
| 41 | 2024-08-22T02:15:31.747Z | INFO | SledAgent (StorageResources): Formatting zpool with generated ID
file = sled-hardware/src/disk.rs:371
id = 97ca59dd-16d5-47a1-9104-ef732f7a459f (zpool)
|
| 42 | 2024-08-22T02:15:31.863Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:307
zpool = ZpoolName { id: 97ca59dd-16d5-47a1-9104-ef732f7a459f (zpool), kind: Internal }
|
| 43 | 2024-08-22T02:15:32.107Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_0.vdev
|
| 44 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 6f31f2f8-a517-49b4-86c9-0d60b67db277 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 97ca59dd-16d5-47a1-9104-ef732f7a459f (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 45 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 46 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 47 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }), tx: ... }
file = sled-storage/src/manager.rs:406
|
| 48 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 49 | 2024-08-22T02:15:32.107Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_1.vdev
|
| 50 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 6f31f2f8-a517-49b4-86c9-0d60b67db277 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 97ca59dd-16d5-47a1-9104-ef732f7a459f (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 51 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }), tx: ... }
file = sled-storage/src/manager.rs:406
|
| 52 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 53 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 54 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 55 | 2024-08-22T02:15:32.107Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_2.vdev
|
| 56 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 6f31f2f8-a517-49b4-86c9-0d60b67db277 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 97ca59dd-16d5-47a1-9104-ef732f7a459f (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 57 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }), tx: ... }
file = sled-storage/src/manager.rs:406
|
| 58 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 59 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 60 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 61 | 2024-08-22T02:15:32.107Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_3.vdev
|
| 62 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 6f31f2f8-a517-49b4-86c9-0d60b67db277 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 97ca59dd-16d5-47a1-9104-ef732f7a459f (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 63 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 64 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 65 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }), tx: ... }
file = sled-storage/src/manager.rs:406
|
| 66 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 67 | 2024-08-22T02:15:32.107Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_4.vdev
|
| 68 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 6f31f2f8-a517-49b4-86c9-0d60b67db277 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 97ca59dd-16d5-47a1-9104-ef732f7a459f (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 69 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 70 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 71 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }), tx: ... }
file = sled-storage/src/manager.rs:406
|
| 72 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 73 | 2024-08-22T02:15:32.107Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_5.vdev
|
| 74 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }), tx: ... }
file = sled-storage/src/manager.rs:406
|
| 75 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 76 | 2024-08-22T02:15:32.107Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_6.vdev
|
| 77 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 6f31f2f8-a517-49b4-86c9-0d60b67db277 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 97ca59dd-16d5-47a1-9104-ef732f7a459f (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 78 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 79 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 80 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }), tx: ... }
file = sled-storage/src/manager.rs:406
|
| 81 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 82 | 2024-08-22T02:15:32.107Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_7.vdev
|
| 83 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 6f31f2f8-a517-49b4-86c9-0d60b67db277 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 97ca59dd-16d5-47a1-9104-ef732f7a459f (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 84 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 85 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 86 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }), tx: ... }
file = sled-storage/src/manager.rs:406
|
| 87 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 88 | 2024-08-22T02:15:32.107Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_8.vdev
|
| 89 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 6f31f2f8-a517-49b4-86c9-0d60b67db277 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 97ca59dd-16d5-47a1-9104-ef732f7a459f (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 90 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }), tx: ... }
file = sled-storage/src/manager.rs:406
|
| 91 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 92 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 93 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 94 | 2024-08-22T02:15:32.107Z | INFO | SledAgent: Waiting for boot disk
file = sled-agent/src/long_running_tasks.rs:97
|
| 95 | 2024-08-22T02:15:32.107Z | INFO | SledAgent: Found boot disk DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
file = sled-agent/src/long_running_tasks.rs:99
|
| 96 | 2024-08-22T02:15:32.107Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 6f31f2f8-a517-49b4-86c9-0d60b67db277 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 97ca59dd-16d5-47a1-9104-ef732f7a459f (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 97 | 2024-08-22T02:15:32.108Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 98 | 2024-08-22T02:15:32.108Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 99 | 2024-08-22T02:15:32.108Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 100 | 2024-08-22T02:15:32.108Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 6f31f2f8-a517-49b4-86c9-0d60b67db277 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 97ca59dd-16d5-47a1-9104-ef732f7a459f (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 101 | 2024-08-22T02:15:32.108Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 102 | 2024-08-22T02:15:32.108Z | INFO | SledAgent: Starting Bootstore
file = sled-agent/src/long_running_tasks.rs:211
|
| 103 | 2024-08-22T02:15:32.108Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 104 | 2024-08-22T02:15:32.108Z | INFO | SledAgent (bootstore): No ledger in /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 105 | 2024-08-22T02:15:32.108Z | INFO | SledAgent (bootstore): No ledger in /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 106 | 2024-08-22T02:15:32.108Z | INFO | SledAgent (bootstore): No ledger found. Loading Fsm::State as Uninitialized
file = bootstore/src/schemes/v0/storage.rs:83
peer_id = pc-sock-i86pc
|
| 107 | 2024-08-22T02:15:32.108Z | INFO | SledAgent (bootstore): No ledger in /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/cluster/bootstore-network-config.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 108 | 2024-08-22T02:15:32.108Z | INFO | SledAgent (bootstore): No ledger in /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/cluster/bootstore-network-config.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 109 | 2024-08-22T02:15:32.108Z | INFO | SledAgent (bootstore): No ledger found for network config
file = bootstore/src/schemes/v0/storage.rs:146
peer_id = pc-sock-i86pc
|
| 110 | 2024-08-22T02:15:32.108Z | INFO | SledAgent: Starting Bootstore DDMD poller
file = sled-agent/src/long_running_tasks.rs:216
|
| 111 | 2024-08-22T02:15:32.108Z | INFO | SledAgent: Starting ZoneBundler related tasks
file = sled-agent/src/long_running_tasks.rs:231
|
| 112 | 2024-08-22T02:15:32.118Z | INFO | SledAgent (ServiceManager): Creating ServiceManager
file = sled-agent/src/services.rs:731
|
| 113 | 2024-08-22T02:15:32.118Z | INFO | SledAgent (auto-cleanup-task): top of bundle cleanup loop
file = sled-agent/src/zone_bundle.rs:186
next_cleanup = Instant { tv_sec: 1468, tv_nsec: 308702914 }
time_to_next_cleanup = 599.99993749s
|
| 114 | 2024-08-22T02:15:32.118Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3797
|
| 115 | 2024-08-22T02:15:32.118Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 116 | 2024-08-22T02:15:32.118Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/sled-agent-request.json
file = common/src/ledger.rs:188
|
| 117 | 2024-08-22T02:15:32.120Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/sled-agent-request.json
file = common/src/ledger.rs:188
|
| 118 | 2024-08-22T02:15:32.121Z | INFO | SledAgent (dropshot (BootstrapAgent)): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
|
| 119 | 2024-08-22T02:15:32.121Z | INFO | SledAgent (SledAgentSprocketsServer): Started listening
file = sled-agent/src/bootstrap/sprockets_server.rs:44
local_addr = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 120 | 2024-08-22T02:15:32.121Z | INFO | SledAgent (RSS): Injecting RSS configuration: RackInitializeRequest {
trust_quorum_peers: None,
bootstrap_discovery: OnlyOurs,
ntp_servers: [
"0.pool.ntp.org",
],
dns_servers: [
1.1.1.1,
9.9.9.9,
],
internal_services_ip_pool_ranges: [
V4(
Ipv4Range {
first: 10.151.2.164,
last: 10.151.2.173,
},
),
],
external_dns_ips: [
10.151.2.164,
10.151.2.165,
],
external_dns_zone_name: "oxide.test",
external_certificates: "<redacted>",
recovery_silo: RecoverySiloConfig {
silo_name: Name(
"recovery",
),
user_name: UserId(
"recovery",
),
user_password_hash: NewPasswordHash(
PasswordHashString {
string: "$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY",
encoding: B64,
},
),
},
rack_network_config: RackNetworkConfigV2 {
rack_subnet: Ipv6Net {
addr: fd00:1122:3344:100::,
width: 56,
},
infra_ip_first: 10.151.2.174,
infra_ip_last: 10.151.2.174,
ports: [
PortConfigV2 {
routes: [
RouteConfig {
destination: V4(
Ipv4Net {
addr: 0.0.0.0,
width: 0,
},
),
nexthop: 10.151.2.100,
vlan_id: None,
},
],
addresses: [
UplinkAddressConfig {
address: V4(
Ipv4Net {
addr: 10.151.2.174,
width: 24,
},
),
vlan_id: None,
},
],
switch: Switch0,
port: "qsfp0",
uplink_port_speed: Speed40G,
uplink_port_fec: None,
bgp_peers: [],
autoneg: false,
},
],
bgp: [],
bfd: [],
},
allowed_source_ips: Any,
}
file = sled-agent/src/rack_setup/service.rs:1018
|
| 121 | 2024-08-22T02:15:32.121Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:73
|
| 122 | 2024-08-22T02:15:32.121Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 123 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (RSS): No ledger in /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/rss-plan-completed.marker
file = common/src/ledger.rs:188
|
| 124 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (RSS): No ledger in /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/rss-plan-completed.marker
file = common/src/ledger.rs:188
|
| 125 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (RSS): RSS configuration has not been fully applied yet
file = sled-agent/src/rack_setup/service.rs:1078
|
| 126 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 127 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (RSS): No ledger in /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/rss-sled-plan.json
file = common/src/ledger.rs:188
|
| 128 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (RSS): No ledger in /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/rss-sled-plan.json
file = common/src/ledger.rs:188
|
| 129 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (RSS): Creating new allocation plan
file = sled-agent/src/rack_setup/service.rs:1119
|
| 130 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (RSS): Serializing plan
file = sled-agent/src/rack_setup/plan/sled.rs:155
|
| 131 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (RSS): Creating plan for the sled at fdb0:18c0:4d0d:a02a::1
file = sled-agent/src/rack_setup/plan/sled.rs:128
|
| 132 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 133 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/.rss-sled-plan.json.tmp
file = common/src/ledger.rs:199
|
| 134 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/.rss-sled-plan.json.tmp
file = common/src/ledger.rs:199
|
| 135 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (RSS): Sled plan written to storage
file = sled-agent/src/rack_setup/plan/sled.rs:175
|
| 136 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (RSS): Writing Rack Network Configuration to bootstore
file = sled-agent/src/rack_setup/service.rs:1152
|
| 137 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 1, current_generation: 0
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
| 138 | 2024-08-22T02:15:32.122Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 139 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 140 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 1
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-sock-i86pc
|
| 141 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (BootstrapAgentRssHandler): Received initialization request from RSS
file = sled-agent/src/bootstrap/rss_handle.rs:217
request = StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 37fb4e95-8c94-4aa8-946d-de6b4770caf9, rack_id: 807d2f15-4cac-40e5-8f79-4faf50cf2768, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 142 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (SledAgentSprocketsServer): Accepted connection
file = sled-agent/src/bootstrap/sprockets_server.rs:66
remote_addr = [fdb0:18c0:4d0d:a02a::1]:64686
|
| 143 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (SledAgentMain): Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 37fb4e95-8c94-4aa8-946d-de6b4770caf9, rack_id: 807d2f15-4cac-40e5-8f79-4faf50cf2768, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }
file = sled-agent/src/bootstrap/server.rs:359
|
| 144 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (SledAgentMain): KeyManager: using hardcoded secret retriever
file = sled-agent/src/bootstrap/server.rs:374
|
| 145 | 2024-08-22T02:15:32.123Z | INFO | SledAgent: setting up sled agent server
file = sled-agent/src/server.rs:44
|
| 146 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (StorageManager): Received KeyManagerReady
file = sled-storage/src/manager.rs:406
|
| 147 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:73
|
| 148 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (StorageManager): Loading storage config
file = sled-storage/src/manager.rs:574
|
| 149 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = load_ledger
|
| 150 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }
|
| 151 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = load_ledger
|
| 152 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (StorageManager): No ledger of physical disks exists
file = sled-storage/src/manager.rs:563
|
| 153 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (StorageManager): KeyManager ready, but no ledger detected
file = sled-storage/src/manager.rs:591
|
| 154 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (StorageManager): automatically managing disks
file = sled-storage/src/manager.rs:395
|
| 155 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (StorageResources): Synchronizing disk managment
file = sled-storage/src/resources.rs:302
|
| 156 | 2024-08-22T02:15:32.123Z | INFO | SledAgent (StorageManager): Successfully synchronized disks without error
file = sled-storage/src/manager.rs:481
result = DisksManagementResult { status: [] }
|
| 157 | 2024-08-22T02:15:32.125Z | INFO | SledAgent (ServiceManager): Enabling switch zone (new)
file = sled-agent/src/services.rs:4031
|
| 158 | 2024-08-22T02:15:32.125Z | INFO | SledAgent (ServiceManager): Starting switch zone
file = sled-agent/src/services.rs:4413
|
| 159 | 2024-08-22T02:15:32.154Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 160 | 2024-08-22T02:15:32.171Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:316
|
| 161 | 2024-08-22T02:15:32.188Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:354
|
| 162 | 2024-08-22T02:15:32.200Z | INFO | SledAgent: SledAgent::new(..) starting
file = sled-agent/src/sled_agent.rs:388
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 163 | 2024-08-22T02:15:32.200Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 164 | 2024-08-22T02:15:32.200Z | INFO | SledAgent: Requested swap device of size 64 GiB
file = sled-agent/src/sled_agent.rs:400
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 165 | 2024-08-22T02:15:32.200Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 166 | 2024-08-22T02:15:32.203Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 81.13248ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "131066f2-9fb3-4bcd-99a5-69d627301ea0", "content-length": "124", "date": "Thu, 22 Aug 2024 02:15:31 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "131066f2-9fb3-4bcd-99a5-69d627301ea0" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 167 | 2024-08-22T02:15:32.213Z | INFO | SledAgent: attempting to create encrypted zvol: name="oxi_6f31f2f8-a517-49b4-86c9-0d60b67db277/swap", size_gb=64
file = sled-agent/src/swap_device.rs:153
|
| 168 | 2024-08-22T02:15:32.246Z | INFO | SledAgent: successfully created encrypted zvol: name="oxi_6f31f2f8-a517-49b4-86c9-0d60b67db277/swap", size_gb=64
file = sled-agent/src/swap_device.rs:249
|
| 169 | 2024-08-22T02:15:32.246Z | INFO | SledAgent: adding swap device: swapname="/dev/zvol/dsk/oxi_6f31f2f8-a517-49b4-86c9-0d60b67db277/swap"
file = sled-agent/src/swap_device.rs:96
|
| 170 | 2024-08-22T02:15:32.251Z | INFO | SledAgent: Mounting backing filesystems
file = sled-agent/src/sled_agent.rs:415
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 171 | 2024-08-22T02:15:32.251Z | INFO | SledAgent (BackingFs): Processing fmd
file = sled-agent/src/backing_fs.rs:126
|
| 172 | 2024-08-22T02:15:32.251Z | INFO | SledAgent (BackingFs): Ensuring dataset oxi_6f31f2f8-a517-49b4-86c9-0d60b67db277/backing/fmd
file = sled-agent/src/backing_fs.rs:136
|
| 173 | 2024-08-22T02:15:32.286Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 174 | 2024-08-22T02:15:32.286Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 65.622466ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "af18511b-fc93-4948-a9ea-30dac831da5d", "content-length": "124", "date": "Thu, 22 Aug 2024 02:15:32 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "af18511b-fc93-4948-a9ea-30dac831da5d" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 175 | 2024-08-22T02:15:32.336Z | INFO | SledAgent (BackingFs): Stopping service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:170
|
| 176 | 2024-08-22T02:15:32.354Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 177 | 2024-08-22T02:15:32.354Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 156.527305ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "00867268-e4c7-40da-b7b4-7f81eeaf761e", "content-length": "124", "date": "Thu, 22 Aug 2024 02:15:32 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "00867268-e4c7-40da-b7b4-7f81eeaf761e" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 178 | 2024-08-22T02:15:32.512Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 179 | 2024-08-22T02:15:32.513Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 182.267531ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "72d79b53-9e00-433b-87a3-b6f8ec578bde", "content-length": "124", "date": "Thu, 22 Aug 2024 02:15:32 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "72d79b53-9e00-433b-87a3-b6f8ec578bde" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 180 | 2024-08-22T02:15:32.555Z | INFO | SledAgent (BackingFs): Mounting oxi_6f31f2f8-a517-49b4-86c9-0d60b67db277/backing/fmd on /var/fm/fmd
file = sled-agent/src/backing_fs.rs:178
|
| 181 | 2024-08-22T02:15:32.568Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/rsrc
file = sled-agent/src/backing_fs.rs:186
|
| 182 | 2024-08-22T02:15:32.568Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/ckpt
file = sled-agent/src/backing_fs.rs:186
|
| 183 | 2024-08-22T02:15:32.568Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/xprt
file = sled-agent/src/backing_fs.rs:186
|
| 184 | 2024-08-22T02:15:32.568Z | INFO | SledAgent (BackingFs): Starting service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:192
|
| 185 | 2024-08-22T02:15:32.697Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 186 | 2024-08-22T02:15:32.697Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 281.863288ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e3f812eb-53a0-40d1-ab97-3fccecde12ea", "content-length": "124", "date": "Thu, 22 Aug 2024 02:15:32 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e3f812eb-53a0-40d1-ab97-3fccecde12ea" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 187 | 2024-08-22T02:15:32.980Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 188 | 2024-08-22T02:15:32.981Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 213.98344ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b54f76b1-1aaf-4f67-a406-fb37d1901723", "content-length": "124", "date": "Thu, 22 Aug 2024 02:15:32 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b54f76b1-1aaf-4f67-a406-fb37d1901723" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 189 | 2024-08-22T02:15:33.196Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 190 | 2024-08-22T02:15:33.197Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 293.172297ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "bc7b673c-f7e1-4f2b-a01c-9ce712aadaeb", "content-length": "124", "date": "Thu, 22 Aug 2024 02:15:32 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "bc7b673c-f7e1-4f2b-a01c-9ce712aadaeb" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 191 | 2024-08-22T02:15:33.302Z | INFO | SledAgent: using '[AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]' as data links for xde driver
file = illumos-utils/src/opte/illumos.rs:90
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 192 | 2024-08-22T02:15:33.307Z | INFO | SledAgent (dropshot): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205
local_addr = [fd00:1122:3344:101::1]:42716
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 193 | 2024-08-22T02:15:33.307Z | INFO | SledAgent (internal-dns-resolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:73
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 194 | 2024-08-22T02:15:33.308Z | INFO | SledAgent (producer-server): starting oximeter metric producer server
address = [fd00:1122:3344:101::1]:42716
file = oximeter/producer/src/lib.rs:283
interval = 30s
producer_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 195 | 2024-08-22T02:15:33.309Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 259.507946ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:391
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 196 | 2024-08-22T02:15:33.310Z | INFO | SledAgent (VmmReservoirManager): 50% of 102952026112 physical ram = 51476013056 bytes)
file = sled-agent/src/vmm_reservoir.rs:267
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 197 | 2024-08-22T02:15:33.310Z | INFO | SledAgent (VmmReservoirManager): Setting reservoir size to 49090 MiB bytes
file = sled-agent/src/vmm_reservoir.rs:275
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 198 | 2024-08-22T02:15:33.310Z | INFO | SledAgent (ServiceManager): sled agent started
file = sled-agent/src/services.rs:915
underlay_address = fd00:1122:3344:101::1
|
| 199 | 2024-08-22T02:15:33.310Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:73
|
| 200 | 2024-08-22T02:15:33.491Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 201 | 2024-08-22T02:15:33.570Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 409.581582ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:391
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 202 | 2024-08-22T02:15:33.981Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 680.384297ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:391
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 203 | 2024-08-22T02:15:34.664Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 2.110011377s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:391
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 204 | 2024-08-22T02:15:35.425Z | INFO | SledAgent (ServiceManager): Setting up dendrite service
file = sled-agent/src/services.rs:2570
|
| 205 | 2024-08-22T02:15:35.425Z | INFO | SledAgent (ServiceManager): Setting up lldpd service
file = sled-agent/src/services.rs:2788
|
| 206 | 2024-08-22T02:15:35.425Z | INFO | SledAgent (ServiceManager): Setting up MGS service
file = sled-agent/src/services.rs:2440
|
| 207 | 2024-08-22T02:15:35.425Z | INFO | SledAgent (ServiceManager): Setting up wicketd service
file = sled-agent/src/services.rs:2487
|
| 208 | 2024-08-22T02:15:35.425Z | INFO | SledAgent (ServiceManager): Setting up mgd service
file = sled-agent/src/services.rs:2869
|
| 209 | 2024-08-22T02:15:35.425Z | INFO | SledAgent (ServiceManager): Setting up mg-ddm service
file = sled-agent/src/services.rs:2911
|
| 210 | 2024-08-22T02:15:35.427Z | INFO | SledAgent (ServiceManager): Setting up tfport service
file = sled-agent/src/services.rs:2713
|
| 211 | 2024-08-22T02:15:35.428Z | INFO | SledAgent (ServiceManager): Setting up Simulated SP service
file = sled-agent/src/services.rs:2481
|
| 212 | 2024-08-22T02:15:35.428Z | INFO | SledAgent (ServiceManager): Profile for oxz_switch:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService0'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='::1'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/mgs">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="address" value='[::1]:12225'/>
<propval type="astring" name="id" value='462d4a63-2441-41eb-9d68-eeea74ed2106'/>
<propval type="astring" name="rack_id" value='807d2f15-4cac-40e5-8f79-4faf50cf2768'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/wicketd">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="address" value='[::1]:12226'/>
<propval type="astring" name="artifact-address" value='[fdb0:18c0:4d0d:a02a::2]:12227'/>
<propval type="astring" name="baseboard-file" value='/opt/oxide/baseboard.json'/>
<propval type="astring" name="mgs-address" value='[::1]:12225'/>
<propval type="astring" name="nexus-proxy-address" value='[::]:12229'/>
<propval type="astring" name="rack-subnet" value='fd00:1122:3344::'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/switch_zone_setup">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="gz_local_link_addr" value='fe80::8:20ff:fe50:9684'/>
<property type="astring" name="link_local_links">
<astring_list>
<value_node value="tfportrear0_0" />
<value_node value="oxBootstrap0" />
</astring_list>
</property>
<propval type="astring" name="bootstrap_addr" value='fdb0:18c0:4d0d:a02a::2'/>
<propval type="astring" name="bootstrap_vnic" value='oxBootstrap0'/>
<propval type="astring" name="baseboard_info" value='{"type":"pc","identifier":"sock","model":"i86pc"}'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/dendrite">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="sled_id" value='37fb4e95-8c94-4aa8-946d-de6b4770caf9'/>
<propval type="astring" name="rack_id" value='807d2f15-4cac-40e5-8f79-4faf50cf2768'/>
<propval type="astring" name="address" value='[::1]:12224'/>
<propval type="astring" name="front_ports" value='1'/>
<propval type="astring" name="rear_ports" value='1'/>
<propval type="astring" name="port_config" value='/opt/oxide/dendrite/misc/softnpu_single_sled_config.toml'/>
<propval type="astring" name="mgmt" value='uds'/>
<propval type="astring" name="uds_path" value='/opt/softnpu/stuff'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/tfport">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="host" value='[::1]'/>
<propval type="astring" name="port" value='12224'/>
<propval type="astring" name="flags" value='--sync-only'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/lldpd">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="board_rev" value='softnpu_front_1_rear_1'/>
<propval type="astring" name="scrimlet_id" value='sock'/>
<propval type="astring" name="scrimlet_model" value='i86pc'/>
<propval type="astring" name="address" value='[::1]:12230'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/pumpkind">
</service>
<service version="1" type="service" name="oxide/mgd">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="sled_uuid" value='37fb4e95-8c94-4aa8-946d-de6b4770caf9'/>
<propval type="astring" name="rack_uuid" value='807d2f15-4cac-40e5-8f79-4faf50cf2768'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/mg-ddm">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="mode" value='transit'/>
<propval type="astring" name="dendrite" value='true'/>
<propval type="astring" name="sled_uuid" value='37fb4e95-8c94-4aa8-946d-de6b4770caf9'/>
<propval type="astring" name="rack_uuid" value='807d2f15-4cac-40e5-8f79-4faf50cf2768'/>
<propval type="astring" name="interfaces" value='tfportrear0_0/ll'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/uplink">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 213 | 2024-08-22T02:15:35.428Z | INFO | SledAgent (ServiceManager): Booting oxz_switch zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_switch
|
| 214 | 2024-08-22T02:15:35.675Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 32.252681ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 215 | 2024-08-22T02:15:35.903Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 52.205226ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 216 | 2024-08-22T02:15:36.079Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 135.813643ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 217 | 2024-08-22T02:15:36.253Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 370.133664ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 218 | 2024-08-22T02:15:36.801Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 869.204901ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 219 | 2024-08-22T02:15:36.802Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 2.523481432s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:391
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 220 | 2024-08-22T02:15:37.690Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 603.996348ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 221 | 2024-08-22T02:15:38.372Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 1.20520758s
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 222 | 2024-08-22T02:15:39.329Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 4.215484377s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:391
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 223 | 2024-08-22T02:15:39.722Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 963.613035ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 224 | 2024-08-22T02:15:40.943Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 509.962095ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 225 | 2024-08-22T02:15:41.913Z | INFO | SledAgent (dropshot (SledAgent)): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205
local_addr = [fd00:1122:3344:101::1]:12345
|
| 226 | 2024-08-22T02:15:41.913Z | INFO | SledAgent (SledAgentMain): Sled Agent loaded; recording configuration
file = sled-agent/src/bootstrap/server.rs:429
|
| 227 | 2024-08-22T02:15:41.913Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 228 | 2024-08-22T02:15:41.913Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:199
|
| 229 | 2024-08-22T02:15:41.914Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:199
|
| 230 | 2024-08-22T02:15:41.914Z | INFO | SledAgent (HardwareMonitor): Sled Agent Started
file = sled-agent/src/hardware_monitor.rs:138
|
| 231 | 2024-08-22T02:15:41.914Z | INFO | SledAgent (SledAgentSprocketsServer): Connection closed
file = sled-agent/src/bootstrap/sprockets_server.rs:73
remote_addr = [fdb0:18c0:4d0d:a02a::1]:64686
|
| 232 | 2024-08-22T02:15:41.914Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
disks = {}
file = sled-agent/src/hardware_monitor.rs:271
underlay_network_info = Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: "qsfp0", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))
|
| 233 | 2024-08-22T02:15:41.914Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3797
|
| 234 | 2024-08-22T02:15:41.914Z | INFO | SledAgent (BootstrapAgentRssHandler): Peer agent initialized
file = sled-agent/src/bootstrap/rss_handle.rs:108
peer = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 235 | 2024-08-22T02:15:41.914Z | INFO | SledAgent (BootstrapAgentRssHandler): Initialized sled agent
file = sled-agent/src/bootstrap/rss_handle.rs:232
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 236 | 2024-08-22T02:15:42.114Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 237 | 2024-08-22T02:15:42.115Z | INFO | SledAgent (RSS): No ledger in /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/rss-service-plan-v3.json
file = common/src/ledger.rs:188
|
| 238 | 2024-08-22T02:15:42.115Z | INFO | SledAgent (RSS): No ledger in /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/rss-service-plan-v3.json
file = common/src/ledger.rs:188
|
| 239 | 2024-08-22T02:15:42.115Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 240 | 2024-08-22T02:15:42.115Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 241 | 2024-08-22T02:15:42.120Z | INFO | SledAgent (ServiceManager): Re-enabling running switch zone (new address)
file = sled-agent/src/services.rs:4055
new = [fd00:1122:3344:101::2, ::1]
old = [::1]
|
| 242 | 2024-08-22T02:15:42.120Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists
file = sled-agent/src/services.rs:4071
|
| 243 | 2024-08-22T02:15:42.120Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:534
zone = oxz_switch
|
| 244 | 2024-08-22T02:15:42.371Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:39416
|
| 245 | 2024-08-22T02:15:42.371Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 246 | 2024-08-22T02:15:42.372Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 265
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:39416
req_id = 15d2f97a-3e20-403b-9ee5-d4e9996455c4
response_code = 200
uri = /inventory
|
| 247 | 2024-08-22T02:15:42.402Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:58235
|
| 248 | 2024-08-22T02:15:42.402Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 71
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:58235
req_id = 85707cf8-dd65-498f-bf0d-549735147fdc
response_code = 200
uri = /sled-role
|
| 249 | 2024-08-22T02:15:42.402Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 250 | 2024-08-22T02:15:42.403Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/.rss-service-plan-v3.json.tmp
file = common/src/ledger.rs:199
|
| 251 | 2024-08-22T02:15:42.403Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/.rss-service-plan-v3.json.tmp
file = common/src/ledger.rs:199
|
| 252 | 2024-08-22T02:15:42.403Z | INFO | SledAgent (RSS): Service plan written to storage
file = sled-agent/src/rack_setup/plan/service.rs:834
|
| 253 | 2024-08-22T02:15:42.436Z | INFO | SledAgent (RSS): attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, id: 50de4b6c-0df7-4cd3-9e5d-6846568ddef3, pool_id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, id: c14f7f08-3e6a-4ccd-afd0-d07d5852e144, pool_id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, id: d7f6452b-6cb0-4eb6-a050-43a1cf3e8931, pool_id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, id: 46a8417e-6c2c-4b23-bbfc-dff964188c18, pool_id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, id: bcf70539-b28a-4f3a-a013-11cd35936e81, pool_id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, id: db9e42cb-715b-47a6-ad13-138ba43a7d8d, pool_id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, id: cf3d2314-1e29-432f-80cb-95eb009c9be4, pool_id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, id: 5bd2feca-e3f9-45ec-a411-8ef88b2d9265, pool_id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, id: fb4af332-e68d-4e89-b185-7658b4d8e606, pool_id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool) }] }
file = sled-agent/src/rack_setup/service.rs:356
sled_address = [fd00:1122:3344:101::1]:12345
|
| 254 | 2024-08-22T02:15:42.436Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:61231
|
| 255 | 2024-08-22T02:15:42.436Z | INFO | SledAgent: physical disks ensure
file = sled-agent/src/sled_agent.rs:833
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 256 | 2024-08-22T02:15:42.436Z | INFO | SledAgent (StorageManager): Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, id: 50de4b6c-0df7-4cd3-9e5d-6846568ddef3, pool_id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, id: c14f7f08-3e6a-4ccd-afd0-d07d5852e144, pool_id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, id: d7f6452b-6cb0-4eb6-a050-43a1cf3e8931, pool_id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, id: 46a8417e-6c2c-4b23-bbfc-dff964188c18, pool_id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, id: bcf70539-b28a-4f3a-a013-11cd35936e81, pool_id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, id: db9e42cb-715b-47a6-ad13-138ba43a7d8d, pool_id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, id: cf3d2314-1e29-432f-80cb-95eb009c9be4, pool_id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, id: 5bd2feca-e3f9-45ec-a411-8ef88b2d9265, pool_id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, id: fb4af332-e68d-4e89-b185-7658b4d8e606, pool_id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool) }] }, tx: ... }
file = sled-storage/src/manager.rs:406
|
| 257 | 2024-08-22T02:15:42.436Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = omicron_physical_disks_ensure
|
| 258 | 2024-08-22T02:15:42.436Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = omicron_physical_disks_ensure
|
| 259 | 2024-08-22T02:15:42.437Z | INFO | SledAgent (StorageManager): No previously-stored 'requested disks', creating new ledger
file = sled-storage/src/manager.rs:645
request = omicron_physical_disks_ensure
|
| 260 | 2024-08-22T02:15:42.437Z | INFO | SledAgent (StorageResources): Synchronizing disk managment
file = sled-storage/src/resources.rs:302
|
| 261 | 2024-08-22T02:15:42.437Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:342
|
| 262 | 2024-08-22T02:15:42.437Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 263 | 2024-08-22T02:15:42.437Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
path = /scratch/u2_0.vdev
|
| 264 | 2024-08-22T02:15:42.598Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev
file = sled-hardware/src/disk.rs:360
|
| 265 | 2024-08-22T02:15:42.598Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:366
id = a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool)
|
| 266 | 2024-08-22T02:15:43.065Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/37fb4e95-8c94-4aa8-946d-de6b4770caf9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:230
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 267 | 2024-08-22T02:15:43.510Z | INFO | SledAgent (VmmReservoirManager): Finished setting reservoir size to 49090 MiB bytes
file = sled-agent/src/vmm_reservoir.rs:279
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 268 | 2024-08-22T02:15:43.547Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 15.162037122s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:391
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 269 | 2024-08-22T02:15:43.556Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:307
zpool = ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }
|
| 270 | 2024-08-22T02:15:43.565Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:351
|
| 271 | 2024-08-22T02:15:43.565Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:353
|
| 272 | 2024-08-22T02:15:43.565Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:357
|
| 273 | 2024-08-22T02:15:43.565Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:359
|
| 274 | 2024-08-22T02:15:43.565Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 275 | 2024-08-22T02:15:43.565Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 276 | 2024-08-22T02:15:43.565Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:371
|
| 277 | 2024-08-22T02:15:43.613Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 278 | 2024-08-22T02:15:43.621Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_a3847786-8a2e-45ee-87a5-075f35fc7eea/crypt/zone
file = sled-storage/src/dataset.rs:414
|
| 279 | 2024-08-22T02:15:43.779Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_a3847786-8a2e-45ee-87a5-075f35fc7eea
file = sled-storage/src/dataset.rs:499
|
| 280 | 2024-08-22T02:15:43.787Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:440
|
| 281 | 2024-08-22T02:15:43.787Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:357
|
| 282 | 2024-08-22T02:15:43.787Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:342
|
| 283 | 2024-08-22T02:15:43.787Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 284 | 2024-08-22T02:15:43.787Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
path = /scratch/u2_1.vdev
|
| 285 | 2024-08-22T02:15:43.798Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev
file = sled-hardware/src/disk.rs:360
|
| 286 | 2024-08-22T02:15:43.798Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:366
id = c613840f-0c10-41fb-9afd-d136750700f5 (zpool)
|
| 287 | 2024-08-22T02:15:43.913Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:307
zpool = ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }
|
| 288 | 2024-08-22T02:15:43.920Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:351
|
| 289 | 2024-08-22T02:15:43.920Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:353
|
| 290 | 2024-08-22T02:15:43.920Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:357
|
| 291 | 2024-08-22T02:15:43.920Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:359
|
| 292 | 2024-08-22T02:15:43.920Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 293 | 2024-08-22T02:15:43.920Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 294 | 2024-08-22T02:15:43.920Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:371
|
| 295 | 2024-08-22T02:15:43.961Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 296 | 2024-08-22T02:15:43.968Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_c613840f-0c10-41fb-9afd-d136750700f5/crypt/zone
file = sled-storage/src/dataset.rs:414
|
| 297 | 2024-08-22T02:15:44.067Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/37fb4e95-8c94-4aa8-946d-de6b4770caf9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:230
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 298 | 2024-08-22T02:15:44.126Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_c613840f-0c10-41fb-9afd-d136750700f5
file = sled-storage/src/dataset.rs:499
|
| 299 | 2024-08-22T02:15:44.133Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:440
|
| 300 | 2024-08-22T02:15:44.134Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:357
|
| 301 | 2024-08-22T02:15:44.134Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:342
|
| 302 | 2024-08-22T02:15:44.134Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 303 | 2024-08-22T02:15:44.134Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
path = /scratch/u2_2.vdev
|
| 304 | 2024-08-22T02:15:44.143Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev
file = sled-hardware/src/disk.rs:360
|
| 305 | 2024-08-22T02:15:44.144Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:366
id = 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool)
|
| 306 | 2024-08-22T02:15:44.266Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/dataset.rs:307
zpool = ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }
|
| 307 | 2024-08-22T02:15:44.274Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/dataset.rs:351
|
| 308 | 2024-08-22T02:15:44.274Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:353
|
| 309 | 2024-08-22T02:15:44.274Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:357
|
| 310 | 2024-08-22T02:15:44.274Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:359
|
| 311 | 2024-08-22T02:15:44.274Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 312 | 2024-08-22T02:15:44.274Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 313 | 2024-08-22T02:15:44.274Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:371
|
| 314 | 2024-08-22T02:15:44.322Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 315 | 2024-08-22T02:15:44.330Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_2beabcd5-0369-41ca-9d09-9a7ad5ca2181/crypt/zone
file = sled-storage/src/dataset.rs:414
|
| 316 | 2024-08-22T02:15:44.486Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_2beabcd5-0369-41ca-9d09-9a7ad5ca2181
file = sled-storage/src/dataset.rs:499
|
| 317 | 2024-08-22T02:15:44.494Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:440
|
| 318 | 2024-08-22T02:15:44.494Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:357
|
| 319 | 2024-08-22T02:15:44.494Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:342
|
| 320 | 2024-08-22T02:15:44.494Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 321 | 2024-08-22T02:15:44.494Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
path = /scratch/u2_3.vdev
|
| 322 | 2024-08-22T02:15:44.504Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev
file = sled-hardware/src/disk.rs:360
|
| 323 | 2024-08-22T02:15:44.504Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:366
id = 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool)
|
| 324 | 2024-08-22T02:15:44.624Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/dataset.rs:307
zpool = ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }
|
| 325 | 2024-08-22T02:15:44.632Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/dataset.rs:351
|
| 326 | 2024-08-22T02:15:44.632Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:353
|
| 327 | 2024-08-22T02:15:44.632Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:357
|
| 328 | 2024-08-22T02:15:44.632Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:359
|
| 329 | 2024-08-22T02:15:44.632Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 330 | 2024-08-22T02:15:44.632Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 331 | 2024-08-22T02:15:44.632Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:371
|
| 332 | 2024-08-22T02:15:44.676Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 333 | 2024-08-22T02:15:44.683Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_817bc2ac-fa69-4e9b-8fd6-6f77d7295e99/crypt/zone
file = sled-storage/src/dataset.rs:414
|
| 334 | 2024-08-22T02:15:44.829Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_817bc2ac-fa69-4e9b-8fd6-6f77d7295e99
file = sled-storage/src/dataset.rs:499
|
| 335 | 2024-08-22T02:15:44.836Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:440
|
| 336 | 2024-08-22T02:15:44.836Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:357
|
| 337 | 2024-08-22T02:15:44.836Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:342
|
| 338 | 2024-08-22T02:15:44.836Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 339 | 2024-08-22T02:15:44.836Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
path = /scratch/u2_4.vdev
|
| 340 | 2024-08-22T02:15:44.846Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev
file = sled-hardware/src/disk.rs:360
|
| 341 | 2024-08-22T02:15:44.846Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:366
id = 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool)
|
| 342 | 2024-08-22T02:15:44.967Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/dataset.rs:307
zpool = ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }
|
| 343 | 2024-08-22T02:15:44.975Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/dataset.rs:351
|
| 344 | 2024-08-22T02:15:44.975Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:353
|
| 345 | 2024-08-22T02:15:44.975Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:357
|
| 346 | 2024-08-22T02:15:44.975Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:359
|
| 347 | 2024-08-22T02:15:44.975Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 348 | 2024-08-22T02:15:44.975Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 349 | 2024-08-22T02:15:44.975Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:371
|
| 350 | 2024-08-22T02:15:45.021Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 351 | 2024-08-22T02:15:45.030Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_50197822-d2db-4c18-aae1-d7d7bdd76af4/crypt/zone
file = sled-storage/src/dataset.rs:414
|
| 352 | 2024-08-22T02:15:45.069Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/37fb4e95-8c94-4aa8-946d-de6b4770caf9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:230
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 353 | 2024-08-22T02:15:45.185Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_50197822-d2db-4c18-aae1-d7d7bdd76af4
file = sled-storage/src/dataset.rs:499
|
| 354 | 2024-08-22T02:15:45.194Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:440
|
| 355 | 2024-08-22T02:15:45.194Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:357
|
| 356 | 2024-08-22T02:15:45.194Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:342
|
| 357 | 2024-08-22T02:15:45.194Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 358 | 2024-08-22T02:15:45.194Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
path = /scratch/u2_5.vdev
|
| 359 | 2024-08-22T02:15:45.204Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev
file = sled-hardware/src/disk.rs:360
|
| 360 | 2024-08-22T02:15:45.204Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:366
id = 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool)
|
| 361 | 2024-08-22T02:15:45.332Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/dataset.rs:307
zpool = ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }
|
| 362 | 2024-08-22T02:15:45.341Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/dataset.rs:351
|
| 363 | 2024-08-22T02:15:45.341Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:353
|
| 364 | 2024-08-22T02:15:45.341Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:357
|
| 365 | 2024-08-22T02:15:45.341Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:359
|
| 366 | 2024-08-22T02:15:45.341Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 367 | 2024-08-22T02:15:45.341Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 368 | 2024-08-22T02:15:45.341Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:371
|
| 369 | 2024-08-22T02:15:45.384Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 370 | 2024-08-22T02:15:45.393Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2/crypt/zone
file = sled-storage/src/dataset.rs:414
|
| 371 | 2024-08-22T02:15:45.547Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2
file = sled-storage/src/dataset.rs:499
|
| 372 | 2024-08-22T02:15:45.555Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:440
|
| 373 | 2024-08-22T02:15:45.555Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:357
|
| 374 | 2024-08-22T02:15:45.555Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:342
|
| 375 | 2024-08-22T02:15:45.555Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 376 | 2024-08-22T02:15:45.555Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
path = /scratch/u2_6.vdev
|
| 377 | 2024-08-22T02:15:45.565Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev
file = sled-hardware/src/disk.rs:360
|
| 378 | 2024-08-22T02:15:45.565Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:366
id = 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool)
|
| 379 | 2024-08-22T02:15:45.689Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/dataset.rs:307
zpool = ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }
|
| 380 | 2024-08-22T02:15:45.697Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/dataset.rs:351
|
| 381 | 2024-08-22T02:15:45.697Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:353
|
| 382 | 2024-08-22T02:15:45.697Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:357
|
| 383 | 2024-08-22T02:15:45.697Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:359
|
| 384 | 2024-08-22T02:15:45.697Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 385 | 2024-08-22T02:15:45.697Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 386 | 2024-08-22T02:15:45.697Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:371
|
| 387 | 2024-08-22T02:15:45.744Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 388 | 2024-08-22T02:15:45.752Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_93a0a2e2-5fff-4e15-9016-1c60e061297f/crypt/zone
file = sled-storage/src/dataset.rs:414
|
| 389 | 2024-08-22T02:15:45.909Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_93a0a2e2-5fff-4e15-9016-1c60e061297f
file = sled-storage/src/dataset.rs:499
|
| 390 | 2024-08-22T02:15:45.917Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:440
|
| 391 | 2024-08-22T02:15:45.917Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:357
|
| 392 | 2024-08-22T02:15:45.917Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:342
|
| 393 | 2024-08-22T02:15:45.917Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 394 | 2024-08-22T02:15:45.917Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
path = /scratch/u2_7.vdev
|
| 395 | 2024-08-22T02:15:45.927Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev
file = sled-hardware/src/disk.rs:360
|
| 396 | 2024-08-22T02:15:45.927Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:366
id = 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool)
|
| 397 | 2024-08-22T02:15:46.046Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/dataset.rs:307
zpool = ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }
|
| 398 | 2024-08-22T02:15:46.054Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/dataset.rs:351
|
| 399 | 2024-08-22T02:15:46.054Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:353
|
| 400 | 2024-08-22T02:15:46.054Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:357
|
| 401 | 2024-08-22T02:15:46.054Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:359
|
| 402 | 2024-08-22T02:15:46.054Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 403 | 2024-08-22T02:15:46.054Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 404 | 2024-08-22T02:15:46.054Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:371
|
| 405 | 2024-08-22T02:15:46.072Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/37fb4e95-8c94-4aa8-946d-de6b4770caf9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:230
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 406 | 2024-08-22T02:15:46.099Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 407 | 2024-08-22T02:15:46.107Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_01770c46-d16d-4f08-bf5d-3bc7b219d504/crypt/zone
file = sled-storage/src/dataset.rs:414
|
| 408 | 2024-08-22T02:15:46.257Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_01770c46-d16d-4f08-bf5d-3bc7b219d504
file = sled-storage/src/dataset.rs:499
|
| 409 | 2024-08-22T02:15:46.265Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:440
|
| 410 | 2024-08-22T02:15:46.265Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:357
|
| 411 | 2024-08-22T02:15:46.265Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:342
|
| 412 | 2024-08-22T02:15:46.265Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 413 | 2024-08-22T02:15:46.265Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
path = /scratch/u2_8.vdev
|
| 414 | 2024-08-22T02:15:46.274Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev
file = sled-hardware/src/disk.rs:360
|
| 415 | 2024-08-22T02:15:46.274Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:366
id = 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool)
|
| 416 | 2024-08-22T02:15:46.392Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/dataset.rs:307
zpool = ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }
|
| 417 | 2024-08-22T02:15:46.401Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/dataset.rs:351
|
| 418 | 2024-08-22T02:15:46.401Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:353
|
| 419 | 2024-08-22T02:15:46.401Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:357
|
| 420 | 2024-08-22T02:15:46.401Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:359
|
| 421 | 2024-08-22T02:15:46.401Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 422 | 2024-08-22T02:15:46.401Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 423 | 2024-08-22T02:15:46.401Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:371
|
| 424 | 2024-08-22T02:15:46.449Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 425 | 2024-08-22T02:15:46.458Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_555e0f8f-2230-434a-8ce6-0ed459a335cf/crypt/zone
file = sled-storage/src/dataset.rs:414
|
| 426 | 2024-08-22T02:15:46.611Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_555e0f8f-2230-434a-8ce6-0ed459a335cf
file = sled-storage/src/dataset.rs:499
|
| 427 | 2024-08-22T02:15:46.619Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:440
|
| 428 | 2024-08-22T02:15:46.619Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:357
|
| 429 | 2024-08-22T02:15:46.619Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/.omicron-physical-disks.json.tmp
file = common/src/ledger.rs:199
request = omicron_physical_disks_ensure
|
| 430 | 2024-08-22T02:15:46.619Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 6f31f2f8-a517-49b4-86c9-0d60b67db277 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 97ca59dd-16d5-47a1-9104-ef732f7a459f (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some("synthetic 1.0")] } }, zpool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 431 | 2024-08-22T02:15:46.620Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 432 | 2024-08-22T02:15:46.620Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 433 | 2024-08-22T02:15:46.620Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/.omicron-physical-disks.json.tmp
file = common/src/ledger.rs:199
request = omicron_physical_disks_ensure
|
| 434 | 2024-08-22T02:15:46.620Z | INFO | SledAgent: physical disks ensure: Updated storage
file = sled-agent/src/sled_agent.rs:837
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 435 | 2024-08-22T02:15:46.620Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 436 | 2024-08-22T02:15:46.620Z | INFO | SledAgent: physical disks ensure: Propagating new generation of disks
file = sled-agent/src/sled_agent.rs:863
generation = Generation(1)
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 437 | 2024-08-22T02:15:46.620Z | INFO | SledAgent: physical disks ensure: Updated storage monitor
file = sled-agent/src/sled_agent.rs:868
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 438 | 2024-08-22T02:15:46.620Z | INFO | SledAgent: physical disks ensure: Updated zone bundler
file = sled-agent/src/sled_agent.rs:873
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 439 | 2024-08-22T02:15:46.620Z | INFO | SledAgent (ProbeManager): use_only_these_disks: Processing new request
file = sled-agent/src/probe_manager.rs:121
gen = Generation(1)
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 440 | 2024-08-22T02:15:46.620Z | INFO | SledAgent: physical disks ensure: Updated probes
file = sled-agent/src/sled_agent.rs:879
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 441 | 2024-08-22T02:15:46.620Z | INFO | SledAgent (InstanceManager): use_only_these_disks: Processing new request
file = sled-agent/src/instance_manager.rs:798
gen = Generation(1)
|
| 442 | 2024-08-22T02:15:46.620Z | INFO | SledAgent: physical disks ensure: Updated instances
file = sled-agent/src/sled_agent.rs:884
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 443 | 2024-08-22T02:15:46.620Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 4183917
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61231
req_id = 9902a751-f112-4d0b-9bbb-04d6606c35fa
response_code = 200
uri = /omicron-physical-disks
|
| 444 | 2024-08-22T02:15:46.652Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 445 | 2024-08-22T02:15:46.653Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:64490
|
| 446 | 2024-08-22T02:15:46.653Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 447 | 2024-08-22T02:15:46.653Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 448 | 2024-08-22T02:15:46.719Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 449 | 2024-08-22T02:15:46.719Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 450 | 2024-08-22T02:15:46.787Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 451 | 2024-08-22T02:15:46.787Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 452 | 2024-08-22T02:15:46.852Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 453 | 2024-08-22T02:15:46.852Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 454 | 2024-08-22T02:15:46.852Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 455 | 2024-08-22T02:15:46.852Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 456 | 2024-08-22T02:15:46.898Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 457 | 2024-08-22T02:15:46.898Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 458 | 2024-08-22T02:15:46.898Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 459 | 2024-08-22T02:15:46.949Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c
file = illumos-utils/src/zone.rs:316
|
| 460 | 2024-08-22T02:15:46.953Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d
file = illumos-utils/src/zone.rs:316
|
| 461 | 2024-08-22T02:15:46.957Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673
file = illumos-utils/src/zone.rs:316
|
| 462 | 2024-08-22T02:15:46.970Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c
file = illumos-utils/src/zone.rs:354
|
| 463 | 2024-08-22T02:15:46.974Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d
file = illumos-utils/src/zone.rs:354
|
| 464 | 2024-08-22T02:15:46.978Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673
file = illumos-utils/src/zone.rs:354
|
| 465 | 2024-08-22T02:15:47.075Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/37fb4e95-8c94-4aa8-946d-de6b4770caf9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:230
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 466 | 2024-08-22T02:15:48.499Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService3'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:3::2'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:3::1'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value='[fd00:1122:3344:3::1]:5353'/>
<propval type="astring" name="dns_address" value='[fd00:1122:3344:3::1]:53'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 467 | 2024-08-22T02:15:48.499Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }
|
| 468 | 2024-08-22T02:15:48.978Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService1'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:1::2'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:1::1'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value='[fd00:1122:3344:1::1]:5353'/>
<propval type="astring" name="dns_address" value='[fd00:1122:3344:1::1]:53'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 469 | 2024-08-22T02:15:49.454Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }
|
| 470 | 2024-08-22T02:15:49.454Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService2'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:2::2'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:2::1'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value='[fd00:1122:3344:2::1]:5353'/>
<propval type="astring" name="dns_address" value='[fd00:1122:3344:2::1]:53'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 471 | 2024-08-22T02:15:49.454Z | INFO | SledAgent (ServiceManager): Booting oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673
|
| 472 | 2024-08-22T02:15:49.457Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }
|
| 473 | 2024-08-22T02:15:49.457Z | INFO | SledAgent (ServiceManager): Booting oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c
|
| 474 | 2024-08-22T02:15:49.462Z | INFO | SledAgent (ServiceManager): Booting oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d
|
| 475 | 2024-08-22T02:15:49.578Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673") failed: Property not found. retry in 54.837236ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673
|
| 476 | 2024-08-22T02:15:49.590Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c") failed: Property not found. retry in 32.751559ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c
|
| 477 | 2024-08-22T02:15:49.596Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d") failed: Property not found. retry in 46.709893ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d
|
| 478 | 2024-08-22T02:15:49.631Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c") failed: Property not found. retry in 81.9756ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c
|
| 479 | 2024-08-22T02:15:49.642Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673") failed: Property not found. retry in 65.053381ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673
|
| 480 | 2024-08-22T02:15:49.669Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d") failed: Property not found. retry in 114.396413ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d
|
| 481 | 2024-08-22T02:15:49.813Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673") failed: Property not found. retry in 249.255412ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673
|
| 482 | 2024-08-22T02:15:49.842Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c") failed: Property not found. retry in 173.72511ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c
|
| 483 | 2024-08-22T02:15:49.874Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d") failed: Property not found. retry in 287.158632ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d
|
| 484 | 2024-08-22T02:15:50.048Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c") failed: Property not found. retry in 592.360286ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c
|
| 485 | 2024-08-22T02:15:50.128Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673") failed: Property not found. retry in 522.307507ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673
|
| 486 | 2024-08-22T02:15:50.180Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d") failed: Property not found. retry in 365.770692ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d
|
| 487 | 2024-08-22T02:15:50.555Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d") failed: Property not found. retry in 955.073229ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d
|
| 488 | 2024-08-22T02:15:50.650Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c") failed: Property not found. retry in 813.936747ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c
|
| 489 | 2024-08-22T02:15:50.658Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673") failed: Property not found. retry in 732.493599ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673
|
| 490 | 2024-08-22T02:15:51.401Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673") failed: Property not found. retry in 694.966301ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673
|
| 491 | 2024-08-22T02:15:51.472Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c") failed: Property not found. retry in 1.030136564s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c
|
| 492 | 2024-08-22T02:15:51.521Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d") failed: Property not found. retry in 685.97917ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d
|
| 493 | 2024-08-22T02:15:51.765Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists - OK
file = sled-agent/src/services.rs:4079
|
| 494 | 2024-08-22T02:15:51.765Z | INFO | SledAgent (ServiceManager): Ensuring there is a default route
file = sled-agent/src/services.rs:4089
gateway = fd00:1122:3344:101::1
|
| 495 | 2024-08-22T02:15:51.773Z | INFO | SledAgent (ServiceManager): configuring dendrite service
file = sled-agent/src/services.rs:4168
|
| 496 | 2024-08-22T02:15:51.913Z | INFO | SledAgent (ServiceManager): refreshed dendrite service with new configuration
file = sled-agent/src/services.rs:4216
|
| 497 | 2024-08-22T02:15:51.913Z | INFO | SledAgent (ServiceManager): configuring lldp service
file = sled-agent/src/services.rs:4243
|
| 498 | 2024-08-22T02:15:51.960Z | INFO | SledAgent (ServiceManager): refreshed lldpd service with new configuration
file = sled-agent/src/services.rs:4256
|
| 499 | 2024-08-22T02:15:51.960Z | INFO | SledAgent (ServiceManager): configuring MGS service
file = sled-agent/src/services.rs:4120
|
| 500 | 2024-08-22T02:15:52.017Z | INFO | SledAgent (ServiceManager): refreshed MGS service with new configuration
file = sled-agent/src/services.rs:4162
|
| 501 | 2024-08-22T02:15:52.017Z | INFO | SledAgent (ServiceManager): configuring wicketd
file = sled-agent/src/services.rs:4223
rack_subnet = fd00:1122:3344::
|
| 502 | 2024-08-22T02:15:52.040Z | INFO | SledAgent (ServiceManager): refreshed wicketd service with new configuration
file = sled-agent/src/services.rs:4234
|
| 503 | 2024-08-22T02:15:52.040Z | INFO | SledAgent (ServiceManager): configuring mgd service
file = sled-agent/src/services.rs:4276
|
| 504 | 2024-08-22T02:15:52.106Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673") failed: Property not found. retry in 721.945404ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673
|
| 505 | 2024-08-22T02:15:52.135Z | INFO | SledAgent (ServiceManager): refreshed mgd service with new configuration
file = sled-agent/src/services.rs:4308
|
| 506 | 2024-08-22T02:15:52.135Z | INFO | SledAgent (ServiceManager): configuring mg-ddm service
file = sled-agent/src/services.rs:4314
|
| 507 | 2024-08-22T02:15:52.216Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d") failed: Property not found. retry in 523.901551ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d
|
| 508 | 2024-08-22T02:15:52.253Z | INFO | SledAgent (ServiceManager): refreshed mg-ddm service with new configuration
file = sled-agent/src/services.rs:4355
|
| 509 | 2024-08-22T02:15:52.253Z | INFO | SledAgent (ServiceManager): Determining physical location of our switch zone at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:366
|
| 510 | 2024-08-22T02:15:52.285Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/37fb4e95-8c94-4aa8-946d-de6b4770caf9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:571
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 511 | 2024-08-22T02:15:52.285Z | WARN | SledAgent (ServiceManager): Failed to get switch ID from MGS (retrying in 50.522415ms)
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/bootstrap/early_networking.rs:385
|
| 512 | 2024-08-22T02:15:52.337Z | WARN | SledAgent (ServiceManager): Failed to get switch ID from MGS (retrying in 57.762012ms)
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/bootstrap/early_networking.rs:385
|
| 513 | 2024-08-22T02:15:52.397Z | INFO | SledAgent (ServiceManager): Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:414
|
| 514 | 2024-08-22T02:15:52.431Z | INFO | SledAgent (ServiceManager): Building Port Configuration
file = sled-agent/src/bootstrap/early_networking.rs:675
|
| 515 | 2024-08-22T02:15:52.431Z | INFO | SledAgent (ServiceManager): Checking dendrite uptime
file = sled-agent/src/bootstrap/early_networking.rs:713
|
| 516 | 2024-08-22T02:15:52.431Z | INFO | SledAgent (ServiceManager): Dendrite online
file = sled-agent/src/bootstrap/early_networking.rs:716
uptime = 0
|
| 517 | 2024-08-22T02:15:52.431Z | INFO | SledAgent (ServiceManager): Configuring default uplink on switch
config = PortSettings {\n links: {\n "0": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}
file = sled-agent/src/bootstrap/early_networking.rs:436
|
| 518 | 2024-08-22T02:15:52.468Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 519 | 2024-08-22T02:15:52.472Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::2]:41878
|
| 520 | 2024-08-22T02:15:52.473Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 704
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::2]:41878
req_id = e35f802c-430e-4e27-8927-0c6b14c8b9dc
response_code = 200
uri = /sled-identifiers
|
| 521 | 2024-08-22T02:15:52.489Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 522 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageManager): Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }
file = sled-storage/src/manager.rs:406
|
| 523 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 524 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 525 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 526 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 527 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 528 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 529 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 530 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 531 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 532 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 533 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 534 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 535 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 536 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 537 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 538 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 539 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 540 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 541 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 542 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 543 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 544 | 2024-08-22T02:15:52.508Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 545 | 2024-08-22T02:15:52.511Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c") failed: Property not found. retry in 1.241981923s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c
|
| 546 | 2024-08-22T02:15:52.750Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d") failed: Property not found. retry in 1.179555941s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d
|
| 547 | 2024-08-22T02:15:52.836Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673") failed: Property not found. retry in 792.081032ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673
|
| 548 | 2024-08-22T02:15:53.637Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673") failed: Property not found. retry in 589.083122ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673
|
| 549 | 2024-08-22T02:15:53.762Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c") failed: Property not found. retry in 1.358762574s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c
|
| 550 | 2024-08-22T02:15:53.939Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d") failed: Property not found. retry in 940.683728ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d
|
| 551 | 2024-08-22T02:15:54.236Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673") failed: Property not found. retry in 1.02391323s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673
|
| 552 | 2024-08-22T02:15:54.889Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d") failed: Property not found. retry in 1.191451269s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d
|
| 553 | 2024-08-22T02:15:56.098Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_internal_dns_0ad4970f-3542-4f24-8bd3-6c96fb94410c
|
| 554 | 2024-08-22T02:15:56.098Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_internal_dns_b57211b8-12b5-4756-ac2b-d3438364d75d
|
| 555 | 2024-08-22T02:15:56.098Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_internal_dns_2662fc93-3f86-483e-b618-470614727673
|
| 556 | 2024-08-22T02:15:56.098Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 557 | 2024-08-22T02:15:56.098Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 558 | 2024-08-22T02:15:56.098Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 9445837
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:64490
req_id = a0ac7a2e-8814-411d-8e6c-0890e62c3954
response_code = 204
uri = /omicron-zones
|
| 559 | 2024-08-22T02:15:56.100Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:1::1]:5353
file = sled-agent/src/rack_setup/service.rs:556
|
| 560 | 2024-08-22T02:15:56.157Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:2::1]:5353
file = sled-agent/src/rack_setup/service.rs:556
|
| 561 | 2024-08-22T02:15:56.212Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:3::1]:5353
file = sled-agent/src/rack_setup/service.rs:556
|
| 562 | 2024-08-22T02:15:56.268Z | INFO | SledAgent (RSS): Configured all DNS servers
file = sled-agent/src/rack_setup/service.rs:592
|
| 563 | 2024-08-22T02:15:56.268Z | INFO | SledAgent (RSS): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:260
|
| 564 | 2024-08-22T02:15:56.300Z | INFO | SledAgent (RSS): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:277
|
| 565 | 2024-08-22T02:15:56.301Z | INFO | SledAgent (RSS): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:302
|
| 566 | 2024-08-22T02:15:56.301Z | INFO | SledAgent (RSS): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:194
|
| 567 | 2024-08-22T02:15:56.330Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 568 | 2024-08-22T02:15:56.330Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:39804
|
| 569 | 2024-08-22T02:15:56.331Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 570 | 2024-08-22T02:15:56.331Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 571 | 2024-08-22T02:15:56.351Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 572 | 2024-08-22T02:15:56.351Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 573 | 2024-08-22T02:15:56.370Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 574 | 2024-08-22T02:15:56.370Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 575 | 2024-08-22T02:15:56.388Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 576 | 2024-08-22T02:15:56.388Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 577 | 2024-08-22T02:15:56.389Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 578 | 2024-08-22T02:15:56.389Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 579 | 2024-08-22T02:15:56.400Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:260
|
| 580 | 2024-08-22T02:15:56.433Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:277
|
| 581 | 2024-08-22T02:15:56.435Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:302
|
| 582 | 2024-08-22T02:15:56.435Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:194
|
| 583 | 2024-08-22T02:15:56.465Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:424
port = Port { inner: PortInner(PortData { name: "opte0", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 150, 21]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }
|
| 584 | 2024-08-22T02:15:56.465Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1235
zone_type = boundary_ntp
|
| 585 | 2024-08-22T02:15:56.466Z | INFO | SledAgent (ServiceManager): no nat entry found for: 10.151.2.169
file = clients/dpd-client/src/lib.rs:83
|
| 586 | 2024-08-22T02:15:56.466Z | INFO | SledAgent (ServiceManager): creating nat entry for: 10.151.2.169
file = clients/dpd-client/src/lib.rs:90
|
| 587 | 2024-08-22T02:15:56.466Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: 10.151.2.169
file = clients/dpd-client/src/lib.rs:118
|
| 588 | 2024-08-22T02:15:56.467Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 589 | 2024-08-22T02:15:56.489Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1
file = illumos-utils/src/zone.rs:316
|
| 590 | 2024-08-22T02:15:56.507Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1
file = illumos-utils/src/zone.rs:354
|
| 591 | 2024-08-22T02:15:57.163Z | INFO | SledAgent (ServiceManager): Profile for oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService4'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::1b'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/chrony-setup">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="allow" value='fd00:1122:3344:100::/56'/>
<propval type="boolean" name="boundary" value='true'/>
<propval type="astring" name="boundary_pool" value='boundary-ntp.control-plane.oxide.internal'/>
<propval type="astring" name="server" value='0.pool.ntp.org'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="1.1.1.1" />
<value_node value="9.9.9.9" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/ntp">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value='opte0'/>
<propval type="astring" name="gateway" value='172.30.3.1'/>
<propval type="astring" name="ip" value='172.30.3.5'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 592 | 2024-08-22T02:15:57.164Z | INFO | SledAgent (ServiceManager): Booting oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1
|
| 593 | 2024-08-22T02:15:57.304Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1") failed: Property not found. retry in 36.879174ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1
|
| 594 | 2024-08-22T02:15:57.349Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1") failed: Property not found. retry in 53.003617ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1
|
| 595 | 2024-08-22T02:15:57.414Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1") failed: Property not found. retry in 276.865736ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1
|
| 596 | 2024-08-22T02:15:57.701Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1") failed: Property not found. retry in 280.167509ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1
|
| 597 | 2024-08-22T02:15:57.990Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1") failed: Property not found. retry in 1.038972359s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1
|
| 598 | 2024-08-22T02:15:59.041Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1") failed: Property not found. retry in 884.28603ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1
|
| 599 | 2024-08-22T02:15:59.935Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1") failed: Property not found. retry in 1.440724589s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1
|
| 600 | 2024-08-22T02:16:01.385Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1") failed: Property not found. retry in 962.036484ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1
|
| 601 | 2024-08-22T02:16:02.369Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1
|
| 602 | 2024-08-22T02:16:02.369Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 603 | 2024-08-22T02:16:02.370Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 604 | 2024-08-22T02:16:02.370Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 6039625
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:39804
req_id = 6bc08417-8279-40a0-ae73-283a060f5932
response_code = 204
uri = /omicron-zones
|
| 605 | 2024-08-22T02:16:02.373Z | INFO | SledAgent (RSS): Waiting for rack time synchronization
file = sled-agent/src/rack_setup/service.rs:633
|
| 606 | 2024-08-22T02:16:02.403Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:613
|
| 607 | 2024-08-22T02:16:02.403Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:43547
|
| 608 | 2024-08-22T02:16:02.412Z | ERRO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3783
|
| 609 | 2024-08-22T02:16:02.412Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 9069
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:43547
req_id = d4f0797f-4cd7-4079-961a-29e7c51263e6
response_code = 500
uri = /timesync
|
| 610 | 2024-08-22T02:16:02.415Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:663
|
| 611 | 2024-08-22T02:16:02.585Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:613
|
| 612 | 2024-08-22T02:16:02.585Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:49141
|
| 613 | 2024-08-22T02:16:06.594Z | ERRO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3783
|
| 614 | 2024-08-22T02:16:06.595Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 4009381
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:49141
req_id = 9eeb0a58-af9f-4aa3-8f92-144f8ae22957
response_code = 500
uri = /timesync
|
| 615 | 2024-08-22T02:16:06.598Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:663
|
| 616 | 2024-08-22T02:16:06.707Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:613
|
| 617 | 2024-08-22T02:16:06.707Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:47928
|
| 618 | 2024-08-22T02:16:06.716Z | ERRO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3783
|
| 619 | 2024-08-22T02:16:06.716Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 8819
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:47928
req_id = f0e94eae-53f6-4f25-9ca1-6e5d300cd5a6
response_code = 500
uri = /timesync
|
| 620 | 2024-08-22T02:16:06.718Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:663
|
| 621 | 2024-08-22T02:16:06.931Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:613
|
| 622 | 2024-08-22T02:16:06.931Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:51085
|
| 623 | 2024-08-22T02:16:06.939Z | ERRO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_d1bb411d-1cd9-4268-99d6-f46af4187aa1': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3783
|
| 624 | 2024-08-22T02:16:06.939Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 7886
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:51085
req_id = e3856360-460a-42f0-86be-1ca5584e60a8
response_code = 500
uri = /timesync
|
| 625 | 2024-08-22T02:16:06.941Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:663
|
| 626 | 2024-08-22T02:16:07.218Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:613
|
| 627 | 2024-08-22T02:16:07.219Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:37430
|
| 628 | 2024-08-22T02:16:08.228Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1008828
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:37430
req_id = 0113a7d2-1d83-464a-b0ae-3cb93aef79d9
response_code = 200
uri = /timesync
|
| 629 | 2024-08-22T02:16:08.231Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:641
|
| 630 | 2024-08-22T02:16:08.231Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:663
|
| 631 | 2024-08-22T02:16:08.442Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:613
|
| 632 | 2024-08-22T02:16:08.442Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:37496
|
| 633 | 2024-08-22T02:16:08.450Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 8435
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:37496
req_id = 69b2eb5f-e3a9-4df4-ad41-730f5d40e5bf
response_code = 200
uri = /timesync
|
| 634 | 2024-08-22T02:16:08.452Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:641
|
| 635 | 2024-08-22T02:16:08.452Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:663
|
| 636 | 2024-08-22T02:16:08.682Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:613
|
| 637 | 2024-08-22T02:16:08.682Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:47078
|
| 638 | 2024-08-22T02:16:08.690Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 7900
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:47078
req_id = 79a56a53-fb05-4335-a2ca-2994760d3418
response_code = 200
uri = /timesync
|
| 639 | 2024-08-22T02:16:08.691Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:641
|
| 640 | 2024-08-22T02:16:08.691Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:663
|
| 641 | 2024-08-22T02:16:08.976Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:613
|
| 642 | 2024-08-22T02:16:08.976Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:46271
|
| 643 | 2024-08-22T02:16:08.984Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 7717
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:46271
req_id = 6b0440fe-a5fe-4e7a-90f7-a846e0d9b8f1
response_code = 200
uri = /timesync
|
| 644 | 2024-08-22T02:16:08.986Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:641
|
| 645 | 2024-08-22T02:16:08.986Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:663
|
| 646 | 2024-08-22T02:16:09.510Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:613
|
| 647 | 2024-08-22T02:16:09.510Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:43970
|
| 648 | 2024-08-22T02:16:09.518Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 7607
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:43970
req_id = 19e743a0-eab2-4f1b-8e63-5e467e778f91
response_code = 200
uri = /timesync
|
| 649 | 2024-08-22T02:16:09.521Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:641
|
| 650 | 2024-08-22T02:16:09.521Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:663
|
| 651 | 2024-08-22T02:16:09.063Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:613
|
| 652 | 2024-08-22T02:16:09.063Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:59331
|
| 653 | 2024-08-22T02:16:09.082Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 18966
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:59331
req_id = 6d7c425d-c93a-4107-95ea-e4f2e033d299
response_code = 200
uri = /timesync
|
| 654 | 2024-08-22T02:16:09.085Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2180654075, ip_addr: 129.250.35.251, stratum: 3, ref_time: 1724292968.907158, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:641
|
| 655 | 2024-08-22T02:16:09.085Z | INFO | SledAgent (RSS): Finished setting up Internal DNS and NTP
file = sled-agent/src/rack_setup/service.rs:1231
|
| 656 | 2024-08-22T02:16:09.116Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 657 | 2024-08-22T02:16:09.116Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:36553
|
| 658 | 2024-08-22T02:16:09.116Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 659 | 2024-08-22T02:16:09.116Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 660 | 2024-08-22T02:16:09.137Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 661 | 2024-08-22T02:16:09.137Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 662 | 2024-08-22T02:16:09.155Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 663 | 2024-08-22T02:16:09.155Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 664 | 2024-08-22T02:16:09.174Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 665 | 2024-08-22T02:16:09.174Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 666 | 2024-08-22T02:16:09.233Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 667 | 2024-08-22T02:16:09.233Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 668 | 2024-08-22T02:16:09.294Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 669 | 2024-08-22T02:16:09.294Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 670 | 2024-08-22T02:16:09.355Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 671 | 2024-08-22T02:16:09.355Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 672 | 2024-08-22T02:16:09.424Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 673 | 2024-08-22T02:16:09.424Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 674 | 2024-08-22T02:16:09.485Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 675 | 2024-08-22T02:16:09.485Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 676 | 2024-08-22T02:16:09.485Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 677 | 2024-08-22T02:16:09.485Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 678 | 2024-08-22T02:16:09.568Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 679 | 2024-08-22T02:16:09.569Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 680 | 2024-08-22T02:16:09.569Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 681 | 2024-08-22T02:16:09.569Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 682 | 2024-08-22T02:16:09.569Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 683 | 2024-08-22T02:16:09.660Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
file = illumos-utils/src/zone.rs:316
|
| 684 | 2024-08-22T02:16:09.664Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
file = illumos-utils/src/zone.rs:316
|
| 685 | 2024-08-22T02:16:09.668Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
file = illumos-utils/src/zone.rs:316
|
| 686 | 2024-08-22T02:16:09.671Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
file = illumos-utils/src/zone.rs:316
|
| 687 | 2024-08-22T02:16:09.676Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
file = illumos-utils/src/zone.rs:316
|
| 688 | 2024-08-22T02:16:09.680Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
file = illumos-utils/src/zone.rs:354
|
| 689 | 2024-08-22T02:16:09.685Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
file = illumos-utils/src/zone.rs:354
|
| 690 | 2024-08-22T02:16:09.690Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
file = illumos-utils/src/zone.rs:354
|
| 691 | 2024-08-22T02:16:09.694Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
file = illumos-utils/src/zone.rs:354
|
| 692 | 2024-08-22T02:16:09.699Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
file = illumos-utils/src/zone.rs:354
|
| 693 | 2024-08-22T02:16:12.769Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService6'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::7'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::7]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='e2f83012-ffa8-48c5-b9f8-a3531cebeca5'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::7]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::7]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 694 | 2024-08-22T02:16:12.770Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 695 | 2024-08-22T02:16:12.776Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService5'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::5'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::5]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='a3ade7be-87ab-4882-beda-b8c4b351c104'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::5]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::5]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 696 | 2024-08-22T02:16:12.776Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 697 | 2024-08-22T02:16:12.784Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService9'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::3'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::3]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='3cd84b19-998e-453f-b3bf-47ee83dcc808'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::3]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::3]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 698 | 2024-08-22T02:16:12.784Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 699 | 2024-08-22T02:16:12.789Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService8'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::6'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::6]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='42ef3e65-0870-4c60-8479-018e44e50e71'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::6]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::6]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 700 | 2024-08-22T02:16:12.789Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 701 | 2024-08-22T02:16:12.800Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService7'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::4'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::4]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='d533f1e0-d3b8-466c-ae7a-c4d68d1d6054'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::4]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::4]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 702 | 2024-08-22T02:16:12.800Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 703 | 2024-08-22T02:16:12.920Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 47.387392ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 704 | 2024-08-22T02:16:12.928Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 70.990697ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 705 | 2024-08-22T02:16:12.935Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5") failed: Property not found. retry in 41.483634ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 706 | 2024-08-22T02:16:12.942Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 43.787055ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 707 | 2024-08-22T02:16:12.949Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 28.57593ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 708 | 2024-08-22T02:16:12.956Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 190.070473ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:424
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 709 | 2024-08-22T02:16:12.975Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 67.202962ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 710 | 2024-08-22T02:16:13.104Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5") failed: Property not found. retry in 56.748079ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 711 | 2024-08-22T02:16:13.149Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 125.579002ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 712 | 2024-08-22T02:16:13.203Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 71.493262ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 713 | 2024-08-22T02:16:13.221Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 121.244632ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 714 | 2024-08-22T02:16:13.249Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 204.744849ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 715 | 2024-08-22T02:16:13.273Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5") failed: Property not found. retry in 260.514756ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 716 | 2024-08-22T02:16:13.300Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 168.676337ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 717 | 2024-08-22T02:16:13.323Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 259.982658ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 718 | 2024-08-22T02:16:13.405Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 281.74974ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 719 | 2024-08-22T02:16:13.504Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 365.103392ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 720 | 2024-08-22T02:16:13.551Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 515.530249ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 721 | 2024-08-22T02:16:13.600Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5") failed: Property not found. retry in 580.735186ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 722 | 2024-08-22T02:16:13.638Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 433.915654ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 723 | 2024-08-22T02:16:13.709Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 365.54141ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 724 | 2024-08-22T02:16:14.105Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 542.469573ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 725 | 2024-08-22T02:16:14.162Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 1.156363177s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 726 | 2024-08-22T02:16:14.340Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 1.034519571s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 727 | 2024-08-22T02:16:14.447Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 581.158919ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 728 | 2024-08-22T02:16:14.545Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5") failed: Property not found. retry in 667.659517ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 729 | 2024-08-22T02:16:14.714Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 704.013076ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 730 | 2024-08-22T02:16:15.091Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 1.322915797s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 731 | 2024-08-22T02:16:15.316Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5") failed: Property not found. retry in 1.330870052s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 732 | 2024-08-22T02:16:15.417Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 921.497065ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 733 | 2024-08-22T02:16:15.439Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 666.952459ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 734 | 2024-08-22T02:16:15.452Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 1.310606971s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 735 | 2024-08-22T02:16:16.118Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 523.984002ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 736 | 2024-08-22T02:16:16.352Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 974.75239ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 737 | 2024-08-22T02:16:16.428Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 846.375839ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 738 | 2024-08-22T02:16:16.654Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 580.763073ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 739 | 2024-08-22T02:16:16.663Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5") failed: Property not found. retry in 1.094195297s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 740 | 2024-08-22T02:16:16.776Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 737.392613ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 741 | 2024-08-22T02:16:17.248Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 740.864508ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 742 | 2024-08-22T02:16:17.285Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 1.093228931s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 743 | 2024-08-22T02:16:17.338Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 579.856508ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 744 | 2024-08-22T02:16:17.525Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 639.375033ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 745 | 2024-08-22T02:16:17.770Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5") failed: Property not found. retry in 1.068765793s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 746 | 2024-08-22T02:16:17.929Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 983.900097ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 747 | 2024-08-22T02:16:18.000Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 940.583353ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 748 | 2024-08-22T02:16:18.176Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 1.275917456s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 749 | 2024-08-22T02:16:18.391Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 1.303072779s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 750 | 2024-08-22T02:16:18.852Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5") failed: Property not found. retry in 719.876523ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 751 | 2024-08-22T02:16:18.925Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 919.86977ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 752 | 2024-08-22T02:16:18.952Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 819.169441ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 753 | 2024-08-22T02:16:19.466Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 1.073380841s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 754 | 2024-08-22T02:16:19.583Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5") failed: Property not found. retry in 1.430061781s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 755 | 2024-08-22T02:16:19.709Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 595.477059ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 756 | 2024-08-22T02:16:19.785Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 1.349146662s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 757 | 2024-08-22T02:16:19.857Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 864.056545ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 758 | 2024-08-22T02:16:20.316Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 615.504514ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 759 | 2024-08-22T02:16:20.553Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 1.26318181s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 760 | 2024-08-22T02:16:20.735Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 1.48723727s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 761 | 2024-08-22T02:16:20.942Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 778.20874ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 762 | 2024-08-22T02:16:21.027Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5") failed: Property not found. retry in 1.074100153s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 763 | 2024-08-22T02:16:21.146Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 983.990741ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 764 | 2024-08-22T02:16:21.221Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:54333
|
| 765 | 2024-08-22T02:16:21.222Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 57
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:54333
req_id = 3b5fca9e-e2e8-4890-b161-53d8d12033ac
response_code = 200
uri = /baseboard
|
| 766 | 2024-08-22T02:16:21.733Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 1.41939905s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 767 | 2024-08-22T02:16:21.830Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 814.28131ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 768 | 2024-08-22T02:16:22.114Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5") failed: Property not found. retry in 1.343665557s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 769 | 2024-08-22T02:16:22.141Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 596.897004ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 770 | 2024-08-22T02:16:22.234Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 775.994762ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 771 | 2024-08-22T02:16:22.657Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 1.139443734s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 772 | 2024-08-22T02:16:22.749Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 543.383268ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 773 | 2024-08-22T02:16:23.022Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 1.351623199s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 774 | 2024-08-22T02:16:23.165Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 528.520719ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 775 | 2024-08-22T02:16:23.304Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 516.915376ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 776 | 2024-08-22T02:16:23.470Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5") failed: Property not found. retry in 1.059055738s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 777 | 2024-08-22T02:16:23.705Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 827.423948ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 778 | 2024-08-22T02:16:23.808Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808") failed: Property not found. retry in 1.283200033s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 779 | 2024-08-22T02:16:23.831Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 537.874773ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 780 | 2024-08-22T02:16:24.380Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104") failed: Property not found. retry in 1.004369275s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 781 | 2024-08-22T02:16:24.387Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054") failed: Property not found. retry in 1.236895034s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 782 | 2024-08-22T02:16:24.554Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71") failed: Property not found. retry in 899.766422ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 783 | 2024-08-22T02:16:25.652Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_cockroachdb_a3ade7be-87ab-4882-beda-b8c4b351c104
|
| 784 | 2024-08-22T02:16:25.652Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_cockroachdb_e2f83012-ffa8-48c5-b9f8-a3531cebeca5
|
| 785 | 2024-08-22T02:16:25.652Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_cockroachdb_d533f1e0-d3b8-466c-ae7a-c4d68d1d6054
|
| 786 | 2024-08-22T02:16:25.652Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_cockroachdb_42ef3e65-0870-4c60-8479-018e44e50e71
|
| 787 | 2024-08-22T02:16:25.652Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_cockroachdb_3cd84b19-998e-453f-b3bf-47ee83dcc808
|
| 788 | 2024-08-22T02:16:25.652Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 789 | 2024-08-22T02:16:25.653Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 790 | 2024-08-22T02:16:25.653Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 16516074
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:36553
req_id = c89b948e-458a-417e-8345-ea14e815abd9
response_code = 204
uri = /omicron-zones
|
| 791 | 2024-08-22T02:16:25.687Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:58428
|
| 792 | 2024-08-22T02:16:25.716Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3617
|
| 793 | 2024-08-22T02:16:28.484Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 412.327012ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:424
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 794 | 2024-08-22T02:16:35.617Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3635
|
| 795 | 2024-08-22T02:16:38.903Z | INFO | SledAgent (ServiceManager): Formatting CRDB - Completed
file = sled-agent/src/services.rs:3658
|
| 796 | 2024-08-22T02:16:38.903Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 13180466
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:58428
req_id = 60b11b61-68cd-42bd-b5c9-1acb37b0d023
response_code = 204
uri = /cockroachdb
|
| 797 | 2024-08-22T02:16:38.940Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 798 | 2024-08-22T02:16:38.941Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:47684
|
| 799 | 2024-08-22T02:16:38.941Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 800 | 2024-08-22T02:16:38.941Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 801 | 2024-08-22T02:16:38.964Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 802 | 2024-08-22T02:16:38.965Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 803 | 2024-08-22T02:16:38.984Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 804 | 2024-08-22T02:16:38.985Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 805 | 2024-08-22T02:16:39.007Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 806 | 2024-08-22T02:16:39.007Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 807 | 2024-08-22T02:16:39.027Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 808 | 2024-08-22T02:16:39.027Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 809 | 2024-08-22T02:16:39.049Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 810 | 2024-08-22T02:16:39.049Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 811 | 2024-08-22T02:16:39.069Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 812 | 2024-08-22T02:16:39.069Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 813 | 2024-08-22T02:16:39.090Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 814 | 2024-08-22T02:16:39.090Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 815 | 2024-08-22T02:16:39.110Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 816 | 2024-08-22T02:16:39.110Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 817 | 2024-08-22T02:16:39.190Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 818 | 2024-08-22T02:16:39.190Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 819 | 2024-08-22T02:16:39.270Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 820 | 2024-08-22T02:16:39.270Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 821 | 2024-08-22T02:16:39.349Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 822 | 2024-08-22T02:16:39.349Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 823 | 2024-08-22T02:16:39.429Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 824 | 2024-08-22T02:16:39.429Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 825 | 2024-08-22T02:16:39.513Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 826 | 2024-08-22T02:16:39.513Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 827 | 2024-08-22T02:16:39.597Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 828 | 2024-08-22T02:16:39.597Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 829 | 2024-08-22T02:16:39.685Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 830 | 2024-08-22T02:16:39.685Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 831 | 2024-08-22T02:16:39.770Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 832 | 2024-08-22T02:16:39.770Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 833 | 2024-08-22T02:16:39.842Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 834 | 2024-08-22T02:16:39.842Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 835 | 2024-08-22T02:16:39.914Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 836 | 2024-08-22T02:16:39.914Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 837 | 2024-08-22T02:16:39.990Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 838 | 2024-08-22T02:16:39.990Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 839 | 2024-08-22T02:16:40.059Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 840 | 2024-08-22T02:16:40.059Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 841 | 2024-08-22T02:16:40.059Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 842 | 2024-08-22T02:16:40.060Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 843 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 844 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 845 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 846 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 847 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 848 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 849 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 850 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 851 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 852 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 853 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 854 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 855 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 856 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 857 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 858 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 859 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 860 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 861 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 862 | 2024-08-22T02:16:40.164Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 1219734
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = a090ca11-f941-409d-a913-8a70c40cb592
response_code = 500
uri = /omicron-zones
|
| 863 | 2024-08-22T02:16:40.164Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a090ca11-f941-409d-a913-8a70c40cb592", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:40 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a090ca11-f941-409d-a913-8a70c40cb592" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 93.08469ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 864 | 2024-08-22T02:16:40.258Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 865 | 2024-08-22T02:16:40.259Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 866 | 2024-08-22T02:16:40.259Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 867 | 2024-08-22T02:16:40.280Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 868 | 2024-08-22T02:16:40.280Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 869 | 2024-08-22T02:16:40.300Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 870 | 2024-08-22T02:16:40.300Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 871 | 2024-08-22T02:16:40.321Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 872 | 2024-08-22T02:16:40.321Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 873 | 2024-08-22T02:16:40.340Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 874 | 2024-08-22T02:16:40.340Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 875 | 2024-08-22T02:16:40.360Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 876 | 2024-08-22T02:16:40.360Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 877 | 2024-08-22T02:16:40.379Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 878 | 2024-08-22T02:16:40.379Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 879 | 2024-08-22T02:16:40.398Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 880 | 2024-08-22T02:16:40.398Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 881 | 2024-08-22T02:16:40.418Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 882 | 2024-08-22T02:16:40.418Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 883 | 2024-08-22T02:16:40.438Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 884 | 2024-08-22T02:16:40.438Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 885 | 2024-08-22T02:16:40.458Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 886 | 2024-08-22T02:16:40.458Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 887 | 2024-08-22T02:16:40.478Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 888 | 2024-08-22T02:16:40.478Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 889 | 2024-08-22T02:16:40.499Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 890 | 2024-08-22T02:16:40.499Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 891 | 2024-08-22T02:16:40.518Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 892 | 2024-08-22T02:16:40.518Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 893 | 2024-08-22T02:16:40.538Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 894 | 2024-08-22T02:16:40.538Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 895 | 2024-08-22T02:16:40.556Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 896 | 2024-08-22T02:16:40.556Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 897 | 2024-08-22T02:16:40.577Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 898 | 2024-08-22T02:16:40.577Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 899 | 2024-08-22T02:16:40.596Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 900 | 2024-08-22T02:16:40.596Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 901 | 2024-08-22T02:16:40.615Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 902 | 2024-08-22T02:16:40.615Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 903 | 2024-08-22T02:16:40.633Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 904 | 2024-08-22T02:16:40.633Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 905 | 2024-08-22T02:16:40.653Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 906 | 2024-08-22T02:16:40.653Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 907 | 2024-08-22T02:16:40.653Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 908 | 2024-08-22T02:16:40.653Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 909 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 910 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 911 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 912 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 913 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 914 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 915 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 916 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 917 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 918 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 919 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 920 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 921 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 922 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 923 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 924 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 925 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 926 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 927 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 928 | 2024-08-22T02:16:40.748Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 488136
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = 1a0d1cac-1c56-48ef-893e-36edfb4a31d8
response_code = 500
uri = /omicron-zones
|
| 929 | 2024-08-22T02:16:40.748Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "1a0d1cac-1c56-48ef-893e-36edfb4a31d8", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:40 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "1a0d1cac-1c56-48ef-893e-36edfb4a31d8" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 145.758897ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 930 | 2024-08-22T02:16:40.895Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 931 | 2024-08-22T02:16:40.895Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 932 | 2024-08-22T02:16:40.895Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 933 | 2024-08-22T02:16:40.920Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 934 | 2024-08-22T02:16:40.920Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 935 | 2024-08-22T02:16:40.940Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 936 | 2024-08-22T02:16:40.940Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 937 | 2024-08-22T02:16:40.963Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 938 | 2024-08-22T02:16:40.963Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 939 | 2024-08-22T02:16:40.985Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 940 | 2024-08-22T02:16:40.985Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 941 | 2024-08-22T02:16:41.006Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 942 | 2024-08-22T02:16:41.006Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 943 | 2024-08-22T02:16:41.027Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 944 | 2024-08-22T02:16:41.027Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 945 | 2024-08-22T02:16:41.046Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 946 | 2024-08-22T02:16:41.046Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 947 | 2024-08-22T02:16:41.068Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 948 | 2024-08-22T02:16:41.068Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 949 | 2024-08-22T02:16:41.087Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 950 | 2024-08-22T02:16:41.087Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 951 | 2024-08-22T02:16:41.107Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 952 | 2024-08-22T02:16:41.107Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 953 | 2024-08-22T02:16:41.125Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 954 | 2024-08-22T02:16:41.126Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 955 | 2024-08-22T02:16:41.144Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 956 | 2024-08-22T02:16:41.144Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 957 | 2024-08-22T02:16:41.165Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 958 | 2024-08-22T02:16:41.165Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 959 | 2024-08-22T02:16:41.186Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 960 | 2024-08-22T02:16:41.186Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 961 | 2024-08-22T02:16:41.207Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 962 | 2024-08-22T02:16:41.207Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 963 | 2024-08-22T02:16:41.229Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 964 | 2024-08-22T02:16:41.229Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 965 | 2024-08-22T02:16:41.248Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 966 | 2024-08-22T02:16:41.248Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 967 | 2024-08-22T02:16:41.269Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 968 | 2024-08-22T02:16:41.269Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 969 | 2024-08-22T02:16:41.289Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 970 | 2024-08-22T02:16:41.289Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 971 | 2024-08-22T02:16:41.307Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 972 | 2024-08-22T02:16:41.308Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 973 | 2024-08-22T02:16:41.308Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 974 | 2024-08-22T02:16:41.308Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 975 | 2024-08-22T02:16:41.402Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 976 | 2024-08-22T02:16:41.402Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 977 | 2024-08-22T02:16:41.402Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 978 | 2024-08-22T02:16:41.402Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 979 | 2024-08-22T02:16:41.402Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 980 | 2024-08-22T02:16:41.402Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 981 | 2024-08-22T02:16:41.402Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 982 | 2024-08-22T02:16:41.402Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 983 | 2024-08-22T02:16:41.402Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 984 | 2024-08-22T02:16:41.402Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 985 | 2024-08-22T02:16:41.402Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 986 | 2024-08-22T02:16:41.402Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 987 | 2024-08-22T02:16:41.402Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 988 | 2024-08-22T02:16:41.402Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 989 | 2024-08-22T02:16:41.403Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 990 | 2024-08-22T02:16:41.403Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 991 | 2024-08-22T02:16:41.403Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 992 | 2024-08-22T02:16:41.403Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 993 | 2024-08-22T02:16:41.403Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 994 | 2024-08-22T02:16:41.403Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 505965
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = dd01c5b8-aff5-479e-9c01-2f45403feb99
response_code = 500
uri = /omicron-zones
|
| 995 | 2024-08-22T02:16:41.403Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "dd01c5b8-aff5-479e-9c01-2f45403feb99", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:40 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "dd01c5b8-aff5-479e-9c01-2f45403feb99" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 204.166785ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 996 | 2024-08-22T02:16:41.609Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 997 | 2024-08-22T02:16:41.609Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 998 | 2024-08-22T02:16:41.609Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 999 | 2024-08-22T02:16:41.631Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1000 | 2024-08-22T02:16:41.631Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1001 | 2024-08-22T02:16:41.649Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1002 | 2024-08-22T02:16:41.649Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1003 | 2024-08-22T02:16:41.667Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1004 | 2024-08-22T02:16:41.667Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1005 | 2024-08-22T02:16:41.688Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1006 | 2024-08-22T02:16:41.688Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1007 | 2024-08-22T02:16:41.708Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1008 | 2024-08-22T02:16:41.709Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1009 | 2024-08-22T02:16:41.728Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1010 | 2024-08-22T02:16:41.728Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1011 | 2024-08-22T02:16:41.745Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1012 | 2024-08-22T02:16:41.745Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1013 | 2024-08-22T02:16:41.764Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1014 | 2024-08-22T02:16:41.764Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1015 | 2024-08-22T02:16:41.782Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1016 | 2024-08-22T02:16:41.782Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1017 | 2024-08-22T02:16:41.802Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1018 | 2024-08-22T02:16:41.802Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1019 | 2024-08-22T02:16:41.819Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1020 | 2024-08-22T02:16:41.819Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1021 | 2024-08-22T02:16:41.842Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1022 | 2024-08-22T02:16:41.842Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1023 | 2024-08-22T02:16:41.862Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1024 | 2024-08-22T02:16:41.862Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1025 | 2024-08-22T02:16:41.879Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1026 | 2024-08-22T02:16:41.879Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1027 | 2024-08-22T02:16:41.899Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1028 | 2024-08-22T02:16:41.899Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1029 | 2024-08-22T02:16:41.919Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1030 | 2024-08-22T02:16:41.919Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1031 | 2024-08-22T02:16:41.939Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1032 | 2024-08-22T02:16:41.939Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1033 | 2024-08-22T02:16:41.962Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1034 | 2024-08-22T02:16:41.962Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1035 | 2024-08-22T02:16:41.983Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1036 | 2024-08-22T02:16:41.983Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1037 | 2024-08-22T02:16:42.003Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1038 | 2024-08-22T02:16:42.003Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1039 | 2024-08-22T02:16:42.003Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1040 | 2024-08-22T02:16:42.004Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1041 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1042 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1043 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1044 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1045 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1046 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1047 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1048 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1049 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1050 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1051 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1052 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1053 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1054 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1055 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1056 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1057 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1058 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1059 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1060 | 2024-08-22T02:16:42.100Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 489816
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = 3df91a04-1d97-4a2f-8ff4-3b3f14db8f8c
response_code = 500
uri = /omicron-zones
|
| 1061 | 2024-08-22T02:16:42.100Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3df91a04-1d97-4a2f-8ff4-3b3f14db8f8c", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:41 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3df91a04-1d97-4a2f-8ff4-3b3f14db8f8c" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 243.352483ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1062 | 2024-08-22T02:16:42.345Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1063 | 2024-08-22T02:16:42.346Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1064 | 2024-08-22T02:16:42.346Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1065 | 2024-08-22T02:16:42.367Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1066 | 2024-08-22T02:16:42.367Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1067 | 2024-08-22T02:16:42.390Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1068 | 2024-08-22T02:16:42.390Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1069 | 2024-08-22T02:16:42.410Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1070 | 2024-08-22T02:16:42.410Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1071 | 2024-08-22T02:16:42.430Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1072 | 2024-08-22T02:16:42.430Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1073 | 2024-08-22T02:16:42.449Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1074 | 2024-08-22T02:16:42.450Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1075 | 2024-08-22T02:16:42.468Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1076 | 2024-08-22T02:16:42.468Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1077 | 2024-08-22T02:16:42.488Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1078 | 2024-08-22T02:16:42.488Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1079 | 2024-08-22T02:16:42.508Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1080 | 2024-08-22T02:16:42.508Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1081 | 2024-08-22T02:16:42.528Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1082 | 2024-08-22T02:16:42.528Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1083 | 2024-08-22T02:16:42.548Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1084 | 2024-08-22T02:16:42.548Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1085 | 2024-08-22T02:16:42.569Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1086 | 2024-08-22T02:16:42.569Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1087 | 2024-08-22T02:16:42.589Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1088 | 2024-08-22T02:16:42.589Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1089 | 2024-08-22T02:16:42.608Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1090 | 2024-08-22T02:16:42.608Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1091 | 2024-08-22T02:16:42.628Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1092 | 2024-08-22T02:16:42.628Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1093 | 2024-08-22T02:16:42.648Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1094 | 2024-08-22T02:16:42.648Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1095 | 2024-08-22T02:16:42.669Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1096 | 2024-08-22T02:16:42.669Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1097 | 2024-08-22T02:16:42.688Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1098 | 2024-08-22T02:16:42.688Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1099 | 2024-08-22T02:16:42.709Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1100 | 2024-08-22T02:16:42.709Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1101 | 2024-08-22T02:16:42.729Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1102 | 2024-08-22T02:16:42.729Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1103 | 2024-08-22T02:16:42.748Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1104 | 2024-08-22T02:16:42.748Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1105 | 2024-08-22T02:16:42.748Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1106 | 2024-08-22T02:16:42.748Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1107 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1108 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1109 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1110 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1111 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1112 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1113 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1114 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1115 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1116 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1117 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1118 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1119 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1120 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1121 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1122 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1123 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1124 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1125 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1126 | 2024-08-22T02:16:42.842Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 495504
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = 9a70ea0b-50b5-4089-9325-1d1632589bbc
response_code = 500
uri = /omicron-zones
|
| 1127 | 2024-08-22T02:16:42.842Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "9a70ea0b-50b5-4089-9325-1d1632589bbc", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:42 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "9a70ea0b-50b5-4089-9325-1d1632589bbc" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 181.427682ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1128 | 2024-08-22T02:16:43.025Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1129 | 2024-08-22T02:16:43.025Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1130 | 2024-08-22T02:16:43.026Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1131 | 2024-08-22T02:16:43.048Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1132 | 2024-08-22T02:16:43.048Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1133 | 2024-08-22T02:16:43.069Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1134 | 2024-08-22T02:16:43.069Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1135 | 2024-08-22T02:16:43.090Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1136 | 2024-08-22T02:16:43.090Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1137 | 2024-08-22T02:16:43.111Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1138 | 2024-08-22T02:16:43.111Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1139 | 2024-08-22T02:16:43.132Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1140 | 2024-08-22T02:16:43.132Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1141 | 2024-08-22T02:16:43.151Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1142 | 2024-08-22T02:16:43.151Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1143 | 2024-08-22T02:16:43.172Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1144 | 2024-08-22T02:16:43.172Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1145 | 2024-08-22T02:16:43.191Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1146 | 2024-08-22T02:16:43.192Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1147 | 2024-08-22T02:16:43.212Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1148 | 2024-08-22T02:16:43.212Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1149 | 2024-08-22T02:16:43.232Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1150 | 2024-08-22T02:16:43.232Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1151 | 2024-08-22T02:16:43.251Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1152 | 2024-08-22T02:16:43.251Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1153 | 2024-08-22T02:16:43.270Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1154 | 2024-08-22T02:16:43.270Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1155 | 2024-08-22T02:16:43.289Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1156 | 2024-08-22T02:16:43.289Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1157 | 2024-08-22T02:16:43.309Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1158 | 2024-08-22T02:16:43.309Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1159 | 2024-08-22T02:16:43.328Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1160 | 2024-08-22T02:16:43.328Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1161 | 2024-08-22T02:16:43.353Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1162 | 2024-08-22T02:16:43.353Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1163 | 2024-08-22T02:16:43.378Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1164 | 2024-08-22T02:16:43.378Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1165 | 2024-08-22T02:16:43.399Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1166 | 2024-08-22T02:16:43.399Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1167 | 2024-08-22T02:16:43.423Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1168 | 2024-08-22T02:16:43.423Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1169 | 2024-08-22T02:16:43.447Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1170 | 2024-08-22T02:16:43.447Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1171 | 2024-08-22T02:16:43.447Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1172 | 2024-08-22T02:16:43.447Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1173 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1174 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1175 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1176 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1177 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1178 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1179 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1180 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1181 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1182 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1183 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1184 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1185 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1186 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1187 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1188 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1189 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1190 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1191 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1192 | 2024-08-22T02:16:43.554Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 527409
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = bf77c54a-39f1-4c06-87d3-008700239f90
response_code = 500
uri = /omicron-zones
|
| 1193 | 2024-08-22T02:16:43.554Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "bf77c54a-39f1-4c06-87d3-008700239f90", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:43 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "bf77c54a-39f1-4c06-87d3-008700239f90" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 148.352612ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1194 | 2024-08-22T02:16:43.704Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1195 | 2024-08-22T02:16:43.705Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1196 | 2024-08-22T02:16:43.705Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1197 | 2024-08-22T02:16:43.729Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1198 | 2024-08-22T02:16:43.729Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1199 | 2024-08-22T02:16:43.752Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1200 | 2024-08-22T02:16:43.752Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1201 | 2024-08-22T02:16:43.775Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1202 | 2024-08-22T02:16:43.775Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1203 | 2024-08-22T02:16:43.796Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1204 | 2024-08-22T02:16:43.796Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1205 | 2024-08-22T02:16:43.819Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1206 | 2024-08-22T02:16:43.819Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1207 | 2024-08-22T02:16:43.840Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1208 | 2024-08-22T02:16:43.840Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1209 | 2024-08-22T02:16:43.861Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1210 | 2024-08-22T02:16:43.861Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1211 | 2024-08-22T02:16:43.882Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1212 | 2024-08-22T02:16:43.882Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1213 | 2024-08-22T02:16:43.904Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1214 | 2024-08-22T02:16:43.904Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1215 | 2024-08-22T02:16:43.927Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1216 | 2024-08-22T02:16:43.927Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1217 | 2024-08-22T02:16:43.947Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1218 | 2024-08-22T02:16:43.947Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1219 | 2024-08-22T02:16:43.971Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1220 | 2024-08-22T02:16:43.971Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1221 | 2024-08-22T02:16:43.993Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1222 | 2024-08-22T02:16:43.993Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1223 | 2024-08-22T02:16:44.019Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1224 | 2024-08-22T02:16:44.019Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1225 | 2024-08-22T02:16:44.042Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1226 | 2024-08-22T02:16:44.042Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1227 | 2024-08-22T02:16:44.062Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1228 | 2024-08-22T02:16:44.062Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1229 | 2024-08-22T02:16:44.083Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1230 | 2024-08-22T02:16:44.083Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1231 | 2024-08-22T02:16:44.105Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1232 | 2024-08-22T02:16:44.105Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1233 | 2024-08-22T02:16:44.125Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1234 | 2024-08-22T02:16:44.125Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1235 | 2024-08-22T02:16:44.146Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1236 | 2024-08-22T02:16:44.146Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1237 | 2024-08-22T02:16:44.146Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1238 | 2024-08-22T02:16:44.146Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1239 | 2024-08-22T02:16:44.246Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 1.156483293s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:424
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 1240 | 2024-08-22T02:16:44.246Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1241 | 2024-08-22T02:16:44.246Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1242 | 2024-08-22T02:16:44.246Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1243 | 2024-08-22T02:16:44.246Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1244 | 2024-08-22T02:16:44.246Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1245 | 2024-08-22T02:16:44.246Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1246 | 2024-08-22T02:16:44.246Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1247 | 2024-08-22T02:16:44.246Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1248 | 2024-08-22T02:16:44.246Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1249 | 2024-08-22T02:16:44.246Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1250 | 2024-08-22T02:16:44.247Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1251 | 2024-08-22T02:16:44.247Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1252 | 2024-08-22T02:16:44.247Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1253 | 2024-08-22T02:16:44.247Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1254 | 2024-08-22T02:16:44.247Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1255 | 2024-08-22T02:16:44.247Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1256 | 2024-08-22T02:16:44.247Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1257 | 2024-08-22T02:16:44.247Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1258 | 2024-08-22T02:16:44.247Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1259 | 2024-08-22T02:16:44.247Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 540968
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = 374eac84-f56f-490d-a18f-0c35299eff94
response_code = 500
uri = /omicron-zones
|
| 1260 | 2024-08-22T02:16:44.247Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "374eac84-f56f-490d-a18f-0c35299eff94", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:44 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "374eac84-f56f-490d-a18f-0c35299eff94" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 355.313108ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1261 | 2024-08-22T02:16:44.604Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1262 | 2024-08-22T02:16:44.604Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1263 | 2024-08-22T02:16:44.604Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1264 | 2024-08-22T02:16:44.630Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1265 | 2024-08-22T02:16:44.630Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1266 | 2024-08-22T02:16:44.653Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1267 | 2024-08-22T02:16:44.653Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1268 | 2024-08-22T02:16:44.673Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1269 | 2024-08-22T02:16:44.673Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1270 | 2024-08-22T02:16:44.693Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1271 | 2024-08-22T02:16:44.693Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1272 | 2024-08-22T02:16:44.713Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1273 | 2024-08-22T02:16:44.713Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1274 | 2024-08-22T02:16:44.734Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1275 | 2024-08-22T02:16:44.734Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1276 | 2024-08-22T02:16:44.754Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1277 | 2024-08-22T02:16:44.754Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1278 | 2024-08-22T02:16:44.773Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1279 | 2024-08-22T02:16:44.773Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1280 | 2024-08-22T02:16:44.796Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1281 | 2024-08-22T02:16:44.796Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1282 | 2024-08-22T02:16:44.816Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1283 | 2024-08-22T02:16:44.816Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1284 | 2024-08-22T02:16:44.838Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1285 | 2024-08-22T02:16:44.838Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1286 | 2024-08-22T02:16:44.859Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1287 | 2024-08-22T02:16:44.859Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1288 | 2024-08-22T02:16:44.882Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1289 | 2024-08-22T02:16:44.882Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1290 | 2024-08-22T02:16:44.901Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1291 | 2024-08-22T02:16:44.901Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1292 | 2024-08-22T02:16:44.922Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1293 | 2024-08-22T02:16:44.922Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1294 | 2024-08-22T02:16:44.941Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1295 | 2024-08-22T02:16:44.942Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1296 | 2024-08-22T02:16:44.960Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1297 | 2024-08-22T02:16:44.960Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1298 | 2024-08-22T02:16:44.981Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1299 | 2024-08-22T02:16:44.981Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1300 | 2024-08-22T02:16:45.001Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1301 | 2024-08-22T02:16:45.001Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1302 | 2024-08-22T02:16:45.025Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1303 | 2024-08-22T02:16:45.025Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1304 | 2024-08-22T02:16:45.025Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1305 | 2024-08-22T02:16:45.025Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1306 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1307 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1308 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1309 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1310 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1311 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1312 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1313 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1314 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1315 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1316 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1317 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1318 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1319 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1320 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1321 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1322 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1323 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1324 | 2024-08-22T02:16:45.118Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1325 | 2024-08-22T02:16:45.118Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 512741
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = 6fa327d0-ef8c-41b3-91c1-2fe014ea5f33
response_code = 500
uri = /omicron-zones
|
| 1326 | 2024-08-22T02:16:45.119Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6fa327d0-ef8c-41b3-91c1-2fe014ea5f33", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:44 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6fa327d0-ef8c-41b3-91c1-2fe014ea5f33" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 257.993154ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1327 | 2024-08-22T02:16:45.379Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1328 | 2024-08-22T02:16:45.379Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1329 | 2024-08-22T02:16:45.379Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1330 | 2024-08-22T02:16:45.402Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1331 | 2024-08-22T02:16:45.402Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1332 | 2024-08-22T02:16:45.425Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1333 | 2024-08-22T02:16:45.425Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1334 | 2024-08-22T02:16:45.446Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1335 | 2024-08-22T02:16:45.446Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1336 | 2024-08-22T02:16:45.468Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1337 | 2024-08-22T02:16:45.468Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1338 | 2024-08-22T02:16:45.488Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1339 | 2024-08-22T02:16:45.488Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1340 | 2024-08-22T02:16:45.509Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1341 | 2024-08-22T02:16:45.509Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1342 | 2024-08-22T02:16:45.530Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1343 | 2024-08-22T02:16:45.530Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1344 | 2024-08-22T02:16:45.549Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1345 | 2024-08-22T02:16:45.549Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1346 | 2024-08-22T02:16:45.570Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1347 | 2024-08-22T02:16:45.570Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1348 | 2024-08-22T02:16:45.592Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1349 | 2024-08-22T02:16:45.592Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1350 | 2024-08-22T02:16:45.613Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1351 | 2024-08-22T02:16:45.613Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1352 | 2024-08-22T02:16:45.633Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1353 | 2024-08-22T02:16:45.633Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1354 | 2024-08-22T02:16:45.653Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1355 | 2024-08-22T02:16:45.653Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1356 | 2024-08-22T02:16:45.676Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1357 | 2024-08-22T02:16:45.676Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1358 | 2024-08-22T02:16:45.699Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1359 | 2024-08-22T02:16:45.699Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1360 | 2024-08-22T02:16:45.721Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1361 | 2024-08-22T02:16:45.721Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1362 | 2024-08-22T02:16:45.739Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1363 | 2024-08-22T02:16:45.739Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1364 | 2024-08-22T02:16:45.759Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1365 | 2024-08-22T02:16:45.759Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1366 | 2024-08-22T02:16:45.779Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1367 | 2024-08-22T02:16:45.779Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1368 | 2024-08-22T02:16:45.797Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1369 | 2024-08-22T02:16:45.797Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1370 | 2024-08-22T02:16:45.797Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1371 | 2024-08-22T02:16:45.797Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1372 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1373 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1374 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1375 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1376 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1377 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1378 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1379 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1380 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1381 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1382 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1383 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1384 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1385 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1386 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1387 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1388 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1389 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1390 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1391 | 2024-08-22T02:16:45.895Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 514936
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = 09e6cf15-f59f-4c07-96bf-3096c299896a
response_code = 500
uri = /omicron-zones
|
| 1392 | 2024-08-22T02:16:45.895Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "09e6cf15-f59f-4c07-96bf-3096c299896a", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:45 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "09e6cf15-f59f-4c07-96bf-3096c299896a" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 382.407034ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1393 | 2024-08-22T02:16:46.280Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1394 | 2024-08-22T02:16:46.281Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1395 | 2024-08-22T02:16:46.281Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1396 | 2024-08-22T02:16:46.303Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1397 | 2024-08-22T02:16:46.303Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1398 | 2024-08-22T02:16:46.323Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1399 | 2024-08-22T02:16:46.323Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1400 | 2024-08-22T02:16:46.345Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1401 | 2024-08-22T02:16:46.345Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1402 | 2024-08-22T02:16:46.367Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1403 | 2024-08-22T02:16:46.367Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1404 | 2024-08-22T02:16:46.388Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1405 | 2024-08-22T02:16:46.388Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1406 | 2024-08-22T02:16:46.411Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1407 | 2024-08-22T02:16:46.411Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1408 | 2024-08-22T02:16:46.431Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1409 | 2024-08-22T02:16:46.431Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1410 | 2024-08-22T02:16:46.450Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1411 | 2024-08-22T02:16:46.450Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1412 | 2024-08-22T02:16:46.468Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1413 | 2024-08-22T02:16:46.468Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1414 | 2024-08-22T02:16:46.490Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1415 | 2024-08-22T02:16:46.490Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1416 | 2024-08-22T02:16:46.512Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1417 | 2024-08-22T02:16:46.512Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1418 | 2024-08-22T02:16:46.533Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1419 | 2024-08-22T02:16:46.533Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1420 | 2024-08-22T02:16:46.554Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1421 | 2024-08-22T02:16:46.554Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1422 | 2024-08-22T02:16:46.574Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1423 | 2024-08-22T02:16:46.574Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1424 | 2024-08-22T02:16:46.594Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1425 | 2024-08-22T02:16:46.595Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1426 | 2024-08-22T02:16:46.615Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1427 | 2024-08-22T02:16:46.615Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1428 | 2024-08-22T02:16:46.636Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1429 | 2024-08-22T02:16:46.636Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1430 | 2024-08-22T02:16:46.657Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1431 | 2024-08-22T02:16:46.657Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1432 | 2024-08-22T02:16:46.679Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1433 | 2024-08-22T02:16:46.679Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1434 | 2024-08-22T02:16:46.699Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1435 | 2024-08-22T02:16:46.699Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1436 | 2024-08-22T02:16:46.699Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1437 | 2024-08-22T02:16:46.699Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1438 | 2024-08-22T02:16:46.795Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1439 | 2024-08-22T02:16:46.795Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1440 | 2024-08-22T02:16:46.795Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1441 | 2024-08-22T02:16:46.795Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1442 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1443 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1444 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1445 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1446 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1447 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1448 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1449 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1450 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1451 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1452 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1453 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1454 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1455 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1456 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1457 | 2024-08-22T02:16:46.796Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 513949
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = a54c7ed7-0f54-4b9c-9af0-bc51c4b38a8f
response_code = 500
uri = /omicron-zones
|
| 1458 | 2024-08-22T02:16:46.796Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a54c7ed7-0f54-4b9c-9af0-bc51c4b38a8f", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:46 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a54c7ed7-0f54-4b9c-9af0-bc51c4b38a8f" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 705.571295ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1459 | 2024-08-22T02:16:47.505Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1460 | 2024-08-22T02:16:47.505Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1461 | 2024-08-22T02:16:47.505Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1462 | 2024-08-22T02:16:47.526Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1463 | 2024-08-22T02:16:47.527Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1464 | 2024-08-22T02:16:47.547Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1465 | 2024-08-22T02:16:47.547Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1466 | 2024-08-22T02:16:47.567Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1467 | 2024-08-22T02:16:47.567Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1468 | 2024-08-22T02:16:47.584Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1469 | 2024-08-22T02:16:47.584Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1470 | 2024-08-22T02:16:47.605Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1471 | 2024-08-22T02:16:47.605Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1472 | 2024-08-22T02:16:47.624Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1473 | 2024-08-22T02:16:47.625Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1474 | 2024-08-22T02:16:47.644Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1475 | 2024-08-22T02:16:47.644Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1476 | 2024-08-22T02:16:47.664Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1477 | 2024-08-22T02:16:47.664Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1478 | 2024-08-22T02:16:47.682Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1479 | 2024-08-22T02:16:47.682Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1480 | 2024-08-22T02:16:47.702Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1481 | 2024-08-22T02:16:47.702Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1482 | 2024-08-22T02:16:47.722Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1483 | 2024-08-22T02:16:47.722Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1484 | 2024-08-22T02:16:47.742Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1485 | 2024-08-22T02:16:47.742Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1486 | 2024-08-22T02:16:47.761Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1487 | 2024-08-22T02:16:47.761Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1488 | 2024-08-22T02:16:47.779Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1489 | 2024-08-22T02:16:47.779Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1490 | 2024-08-22T02:16:47.798Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1491 | 2024-08-22T02:16:47.798Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1492 | 2024-08-22T02:16:47.815Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1493 | 2024-08-22T02:16:47.815Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1494 | 2024-08-22T02:16:47.833Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1495 | 2024-08-22T02:16:47.833Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1496 | 2024-08-22T02:16:47.854Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1497 | 2024-08-22T02:16:47.854Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1498 | 2024-08-22T02:16:47.872Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1499 | 2024-08-22T02:16:47.872Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1500 | 2024-08-22T02:16:47.891Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1501 | 2024-08-22T02:16:47.891Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1502 | 2024-08-22T02:16:47.891Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1503 | 2024-08-22T02:16:47.892Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1504 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1505 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1506 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1507 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1508 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1509 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1510 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1511 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1512 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1513 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1514 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1515 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1516 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1517 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1518 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1519 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1520 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1521 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1522 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1523 | 2024-08-22T02:16:47.985Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 478639
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = 491244a9-c564-41c6-ab7f-a50c4102d0dc
response_code = 500
uri = /omicron-zones
|
| 1524 | 2024-08-22T02:16:47.985Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "491244a9-c564-41c6-ab7f-a50c4102d0dc", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:47 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "491244a9-c564-41c6-ab7f-a50c4102d0dc" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 396.299547ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1525 | 2024-08-22T02:16:48.384Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1526 | 2024-08-22T02:16:48.385Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1527 | 2024-08-22T02:16:48.385Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1528 | 2024-08-22T02:16:48.406Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1529 | 2024-08-22T02:16:48.406Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1530 | 2024-08-22T02:16:48.426Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1531 | 2024-08-22T02:16:48.426Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1532 | 2024-08-22T02:16:48.445Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1533 | 2024-08-22T02:16:48.445Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1534 | 2024-08-22T02:16:48.468Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1535 | 2024-08-22T02:16:48.468Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1536 | 2024-08-22T02:16:48.488Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1537 | 2024-08-22T02:16:48.488Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1538 | 2024-08-22T02:16:48.507Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1539 | 2024-08-22T02:16:48.507Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1540 | 2024-08-22T02:16:48.526Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1541 | 2024-08-22T02:16:48.526Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1542 | 2024-08-22T02:16:48.545Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1543 | 2024-08-22T02:16:48.545Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1544 | 2024-08-22T02:16:48.566Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1545 | 2024-08-22T02:16:48.566Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1546 | 2024-08-22T02:16:48.587Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1547 | 2024-08-22T02:16:48.588Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1548 | 2024-08-22T02:16:48.608Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1549 | 2024-08-22T02:16:48.608Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1550 | 2024-08-22T02:16:48.629Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1551 | 2024-08-22T02:16:48.629Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1552 | 2024-08-22T02:16:48.649Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1553 | 2024-08-22T02:16:48.649Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1554 | 2024-08-22T02:16:48.667Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1555 | 2024-08-22T02:16:48.667Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1556 | 2024-08-22T02:16:48.687Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1557 | 2024-08-22T02:16:48.687Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1558 | 2024-08-22T02:16:48.708Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1559 | 2024-08-22T02:16:48.708Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1560 | 2024-08-22T02:16:48.727Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1561 | 2024-08-22T02:16:48.727Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1562 | 2024-08-22T02:16:48.746Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1563 | 2024-08-22T02:16:48.746Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1564 | 2024-08-22T02:16:48.766Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1565 | 2024-08-22T02:16:48.766Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1566 | 2024-08-22T02:16:48.785Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1567 | 2024-08-22T02:16:48.785Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1568 | 2024-08-22T02:16:48.785Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1569 | 2024-08-22T02:16:48.785Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1570 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1571 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1572 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1573 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1574 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1575 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1576 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1577 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1578 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1579 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1580 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1581 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1582 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1583 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1584 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1585 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1586 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1587 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1588 | 2024-08-22T02:16:48.880Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1589 | 2024-08-22T02:16:48.881Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 495521
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = ed37d070-a153-49f8-bfca-0b3f882797b7
response_code = 500
uri = /omicron-zones
|
| 1590 | 2024-08-22T02:16:48.881Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "ed37d070-a153-49f8-bfca-0b3f882797b7", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:48 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "ed37d070-a153-49f8-bfca-0b3f882797b7" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 855.331688ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1591 | 2024-08-22T02:16:49.739Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1592 | 2024-08-22T02:16:49.740Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1593 | 2024-08-22T02:16:49.740Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1594 | 2024-08-22T02:16:49.762Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1595 | 2024-08-22T02:16:49.762Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1596 | 2024-08-22T02:16:49.781Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1597 | 2024-08-22T02:16:49.781Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1598 | 2024-08-22T02:16:49.800Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1599 | 2024-08-22T02:16:49.800Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1600 | 2024-08-22T02:16:49.819Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1601 | 2024-08-22T02:16:49.819Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1602 | 2024-08-22T02:16:49.837Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1603 | 2024-08-22T02:16:49.837Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1604 | 2024-08-22T02:16:49.856Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1605 | 2024-08-22T02:16:49.856Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1606 | 2024-08-22T02:16:49.875Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1607 | 2024-08-22T02:16:49.875Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1608 | 2024-08-22T02:16:49.894Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1609 | 2024-08-22T02:16:49.894Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1610 | 2024-08-22T02:16:49.915Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1611 | 2024-08-22T02:16:49.915Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1612 | 2024-08-22T02:16:49.934Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1613 | 2024-08-22T02:16:49.934Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1614 | 2024-08-22T02:16:49.954Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1615 | 2024-08-22T02:16:49.954Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1616 | 2024-08-22T02:16:49.974Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1617 | 2024-08-22T02:16:49.974Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1618 | 2024-08-22T02:16:49.992Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1619 | 2024-08-22T02:16:49.992Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1620 | 2024-08-22T02:16:50.012Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1621 | 2024-08-22T02:16:50.012Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1622 | 2024-08-22T02:16:50.030Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1623 | 2024-08-22T02:16:50.030Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1624 | 2024-08-22T02:16:50.048Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1625 | 2024-08-22T02:16:50.048Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1626 | 2024-08-22T02:16:50.068Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1627 | 2024-08-22T02:16:50.068Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1628 | 2024-08-22T02:16:50.087Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1629 | 2024-08-22T02:16:50.087Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1630 | 2024-08-22T02:16:50.107Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1631 | 2024-08-22T02:16:50.107Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1632 | 2024-08-22T02:16:50.127Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1633 | 2024-08-22T02:16:50.127Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1634 | 2024-08-22T02:16:50.128Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1635 | 2024-08-22T02:16:50.128Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1636 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1637 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1638 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1639 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1640 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1641 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1642 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1643 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1644 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1645 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1646 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1647 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1648 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1649 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1650 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1651 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1652 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1653 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1654 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1655 | 2024-08-22T02:16:50.223Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 482191
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = 41646586-1c59-4297-a7f7-bc46fdd5fab9
response_code = 500
uri = /omicron-zones
|
| 1656 | 2024-08-22T02:16:50.223Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "41646586-1c59-4297-a7f7-bc46fdd5fab9", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:49 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "41646586-1c59-4297-a7f7-bc46fdd5fab9" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 615.506062ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1657 | 2024-08-22T02:16:50.842Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1658 | 2024-08-22T02:16:50.842Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1659 | 2024-08-22T02:16:50.842Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1660 | 2024-08-22T02:16:50.864Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1661 | 2024-08-22T02:16:50.864Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1662 | 2024-08-22T02:16:50.883Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1663 | 2024-08-22T02:16:50.883Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1664 | 2024-08-22T02:16:50.904Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1665 | 2024-08-22T02:16:50.904Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1666 | 2024-08-22T02:16:50.924Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1667 | 2024-08-22T02:16:50.924Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1668 | 2024-08-22T02:16:50.943Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1669 | 2024-08-22T02:16:50.943Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1670 | 2024-08-22T02:16:50.961Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1671 | 2024-08-22T02:16:50.961Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1672 | 2024-08-22T02:16:50.980Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1673 | 2024-08-22T02:16:50.980Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1674 | 2024-08-22T02:16:50.998Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1675 | 2024-08-22T02:16:50.998Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1676 | 2024-08-22T02:16:51.018Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1677 | 2024-08-22T02:16:51.019Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1678 | 2024-08-22T02:16:51.037Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1679 | 2024-08-22T02:16:51.037Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1680 | 2024-08-22T02:16:51.058Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1681 | 2024-08-22T02:16:51.058Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1682 | 2024-08-22T02:16:51.076Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1683 | 2024-08-22T02:16:51.076Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1684 | 2024-08-22T02:16:51.097Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1685 | 2024-08-22T02:16:51.097Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1686 | 2024-08-22T02:16:51.117Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1687 | 2024-08-22T02:16:51.117Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1688 | 2024-08-22T02:16:51.136Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1689 | 2024-08-22T02:16:51.136Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1690 | 2024-08-22T02:16:51.158Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1691 | 2024-08-22T02:16:51.158Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1692 | 2024-08-22T02:16:51.177Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1693 | 2024-08-22T02:16:51.177Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1694 | 2024-08-22T02:16:51.197Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1695 | 2024-08-22T02:16:51.197Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1696 | 2024-08-22T02:16:51.217Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1697 | 2024-08-22T02:16:51.217Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1698 | 2024-08-22T02:16:51.236Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1699 | 2024-08-22T02:16:51.236Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1700 | 2024-08-22T02:16:51.236Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1701 | 2024-08-22T02:16:51.236Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1702 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1703 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1704 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1705 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1706 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1707 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1708 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1709 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1710 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1711 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1712 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1713 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1714 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1715 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1716 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1717 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1718 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1719 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1720 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1721 | 2024-08-22T02:16:51.332Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 488718
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = d790818f-cc26-4c59-b52c-cd80f6c030f1
response_code = 500
uri = /omicron-zones
|
| 1722 | 2024-08-22T02:16:51.332Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d790818f-cc26-4c59-b52c-cd80f6c030f1", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:50 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d790818f-cc26-4c59-b52c-cd80f6c030f1" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 1.330229807s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1723 | 2024-08-22T02:16:51.343Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56350
|
| 1724 | 2024-08-22T02:16:51.344Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 690
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56350
req_id = 0fd998c2-3c71-40b3-a01c-cedc38f8da51
response_code = 200
uri = /baseboard
|
| 1725 | 2024-08-22T02:16:52.666Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1726 | 2024-08-22T02:16:52.667Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1727 | 2024-08-22T02:16:52.667Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1728 | 2024-08-22T02:16:52.687Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1729 | 2024-08-22T02:16:52.687Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1730 | 2024-08-22T02:16:52.707Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1731 | 2024-08-22T02:16:52.707Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1732 | 2024-08-22T02:16:52.728Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1733 | 2024-08-22T02:16:52.728Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1734 | 2024-08-22T02:16:52.747Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1735 | 2024-08-22T02:16:52.747Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1736 | 2024-08-22T02:16:52.769Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1737 | 2024-08-22T02:16:52.769Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1738 | 2024-08-22T02:16:52.789Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1739 | 2024-08-22T02:16:52.789Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1740 | 2024-08-22T02:16:52.809Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1741 | 2024-08-22T02:16:52.809Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1742 | 2024-08-22T02:16:52.829Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1743 | 2024-08-22T02:16:52.829Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1744 | 2024-08-22T02:16:52.849Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1745 | 2024-08-22T02:16:52.849Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1746 | 2024-08-22T02:16:52.871Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1747 | 2024-08-22T02:16:52.871Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1748 | 2024-08-22T02:16:52.891Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1749 | 2024-08-22T02:16:52.891Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1750 | 2024-08-22T02:16:52.912Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1751 | 2024-08-22T02:16:52.912Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1752 | 2024-08-22T02:16:52.932Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1753 | 2024-08-22T02:16:52.932Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1754 | 2024-08-22T02:16:52.951Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1755 | 2024-08-22T02:16:52.951Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1756 | 2024-08-22T02:16:52.971Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1757 | 2024-08-22T02:16:52.971Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1758 | 2024-08-22T02:16:52.990Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1759 | 2024-08-22T02:16:52.990Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1760 | 2024-08-22T02:16:53.010Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1761 | 2024-08-22T02:16:53.010Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1762 | 2024-08-22T02:16:53.031Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1763 | 2024-08-22T02:16:53.031Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1764 | 2024-08-22T02:16:53.051Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1765 | 2024-08-22T02:16:53.051Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1766 | 2024-08-22T02:16:53.070Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1767 | 2024-08-22T02:16:53.070Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1768 | 2024-08-22T02:16:53.070Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1769 | 2024-08-22T02:16:53.070Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1770 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1771 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1772 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1773 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1774 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1775 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1776 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1777 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1778 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1779 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1780 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1781 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1782 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1783 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1784 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1785 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1786 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1787 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1788 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1789 | 2024-08-22T02:16:53.168Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 500370
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = 2cd0f9e2-bcb9-4124-9290-45582c667f17
response_code = 500
uri = /omicron-zones
|
| 1790 | 2024-08-22T02:16:53.168Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "2cd0f9e2-bcb9-4124-9290-45582c667f17", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:52 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "2cd0f9e2-bcb9-4124-9290-45582c667f17" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 1.287399206s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1791 | 2024-08-22T02:16:54.460Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1792 | 2024-08-22T02:16:54.460Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1793 | 2024-08-22T02:16:54.460Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1794 | 2024-08-22T02:16:54.484Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1795 | 2024-08-22T02:16:54.484Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1796 | 2024-08-22T02:16:54.502Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1797 | 2024-08-22T02:16:54.502Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1798 | 2024-08-22T02:16:54.521Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1799 | 2024-08-22T02:16:54.521Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1800 | 2024-08-22T02:16:54.540Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1801 | 2024-08-22T02:16:54.540Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1802 | 2024-08-22T02:16:54.561Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1803 | 2024-08-22T02:16:54.561Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1804 | 2024-08-22T02:16:54.579Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1805 | 2024-08-22T02:16:54.579Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1806 | 2024-08-22T02:16:54.600Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1807 | 2024-08-22T02:16:54.600Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1808 | 2024-08-22T02:16:54.619Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1809 | 2024-08-22T02:16:54.619Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1810 | 2024-08-22T02:16:54.639Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1811 | 2024-08-22T02:16:54.639Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1812 | 2024-08-22T02:16:54.661Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1813 | 2024-08-22T02:16:54.661Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1814 | 2024-08-22T02:16:54.682Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1815 | 2024-08-22T02:16:54.682Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1816 | 2024-08-22T02:16:54.702Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1817 | 2024-08-22T02:16:54.702Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1818 | 2024-08-22T02:16:54.722Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1819 | 2024-08-22T02:16:54.722Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1820 | 2024-08-22T02:16:54.742Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1821 | 2024-08-22T02:16:54.742Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1822 | 2024-08-22T02:16:54.761Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1823 | 2024-08-22T02:16:54.761Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1824 | 2024-08-22T02:16:54.783Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1825 | 2024-08-22T02:16:54.783Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1826 | 2024-08-22T02:16:54.806Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1827 | 2024-08-22T02:16:54.806Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1828 | 2024-08-22T02:16:54.825Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1829 | 2024-08-22T02:16:54.825Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1830 | 2024-08-22T02:16:54.847Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1831 | 2024-08-22T02:16:54.847Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1832 | 2024-08-22T02:16:54.867Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1833 | 2024-08-22T02:16:54.867Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1834 | 2024-08-22T02:16:54.868Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1835 | 2024-08-22T02:16:54.868Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1836 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1837 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1838 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1839 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1840 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1841 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1842 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1843 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1844 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1845 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1846 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1847 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1848 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1849 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1850 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1851 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1852 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1853 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1854 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1855 | 2024-08-22T02:16:54.965Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 503748
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = c2bc657e-b5d3-4220-9c39-663d59060ea8
response_code = 500
uri = /omicron-zones
|
| 1856 | 2024-08-22T02:16:54.965Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "c2bc657e-b5d3-4220-9c39-663d59060ea8", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:54 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "c2bc657e-b5d3-4220-9c39-663d59060ea8" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 1.888905655s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1857 | 2024-08-22T02:16:56.861Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1858 | 2024-08-22T02:16:56.862Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1859 | 2024-08-22T02:16:56.862Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1860 | 2024-08-22T02:16:56.885Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1861 | 2024-08-22T02:16:56.885Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1862 | 2024-08-22T02:16:56.905Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1863 | 2024-08-22T02:16:56.905Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1864 | 2024-08-22T02:16:56.926Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1865 | 2024-08-22T02:16:56.926Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1866 | 2024-08-22T02:16:56.945Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1867 | 2024-08-22T02:16:56.945Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1868 | 2024-08-22T02:16:56.966Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1869 | 2024-08-22T02:16:56.966Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1870 | 2024-08-22T02:16:56.986Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1871 | 2024-08-22T02:16:56.986Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1872 | 2024-08-22T02:16:57.005Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1873 | 2024-08-22T02:16:57.005Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1874 | 2024-08-22T02:16:57.026Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1875 | 2024-08-22T02:16:57.026Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1876 | 2024-08-22T02:16:57.045Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1877 | 2024-08-22T02:16:57.045Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1878 | 2024-08-22T02:16:57.066Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1879 | 2024-08-22T02:16:57.066Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1880 | 2024-08-22T02:16:57.086Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1881 | 2024-08-22T02:16:57.087Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1882 | 2024-08-22T02:16:57.106Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1883 | 2024-08-22T02:16:57.106Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1884 | 2024-08-22T02:16:57.128Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1885 | 2024-08-22T02:16:57.128Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1886 | 2024-08-22T02:16:57.147Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1887 | 2024-08-22T02:16:57.147Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1888 | 2024-08-22T02:16:57.167Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1889 | 2024-08-22T02:16:57.167Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1890 | 2024-08-22T02:16:57.188Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1891 | 2024-08-22T02:16:57.188Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1892 | 2024-08-22T02:16:57.209Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1893 | 2024-08-22T02:16:57.209Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1894 | 2024-08-22T02:16:57.231Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1895 | 2024-08-22T02:16:57.231Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1896 | 2024-08-22T02:16:57.249Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1897 | 2024-08-22T02:16:57.249Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1898 | 2024-08-22T02:16:57.270Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1899 | 2024-08-22T02:16:57.270Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1900 | 2024-08-22T02:16:57.270Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1901 | 2024-08-22T02:16:57.270Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1902 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1903 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1904 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1905 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1906 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1907 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1908 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1909 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1910 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1911 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1912 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1913 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1914 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1915 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1916 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1917 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1918 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1919 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1920 | 2024-08-22T02:16:57.366Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1921 | 2024-08-22T02:16:57.366Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 503986
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = a6ffbf1b-0b1d-4c55-b350-0e03e70844c5
response_code = 500
uri = /omicron-zones
|
| 1922 | 2024-08-22T02:16:57.367Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a6ffbf1b-0b1d-4c55-b350-0e03e70844c5", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:56 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a6ffbf1b-0b1d-4c55-b350-0e03e70844c5" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 1.687207398s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1923 | 2024-08-22T02:16:59.060Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1924 | 2024-08-22T02:16:59.061Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1925 | 2024-08-22T02:16:59.061Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1926 | 2024-08-22T02:16:59.083Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1927 | 2024-08-22T02:16:59.083Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1928 | 2024-08-22T02:16:59.103Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1929 | 2024-08-22T02:16:59.103Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1930 | 2024-08-22T02:16:59.122Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1931 | 2024-08-22T02:16:59.122Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1932 | 2024-08-22T02:16:59.143Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1933 | 2024-08-22T02:16:59.143Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1934 | 2024-08-22T02:16:59.162Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1935 | 2024-08-22T02:16:59.162Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1936 | 2024-08-22T02:16:59.182Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1937 | 2024-08-22T02:16:59.182Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1938 | 2024-08-22T02:16:59.201Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1939 | 2024-08-22T02:16:59.201Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1940 | 2024-08-22T02:16:59.219Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1941 | 2024-08-22T02:16:59.219Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1942 | 2024-08-22T02:16:59.240Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1943 | 2024-08-22T02:16:59.240Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1944 | 2024-08-22T02:16:59.261Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1945 | 2024-08-22T02:16:59.261Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1946 | 2024-08-22T02:16:59.280Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1947 | 2024-08-22T02:16:59.280Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1948 | 2024-08-22T02:16:59.301Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1949 | 2024-08-22T02:16:59.301Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1950 | 2024-08-22T02:16:59.320Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1951 | 2024-08-22T02:16:59.320Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1952 | 2024-08-22T02:16:59.340Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1953 | 2024-08-22T02:16:59.340Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1954 | 2024-08-22T02:16:59.359Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1955 | 2024-08-22T02:16:59.359Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1956 | 2024-08-22T02:16:59.379Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1957 | 2024-08-22T02:16:59.379Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1958 | 2024-08-22T02:16:59.398Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1959 | 2024-08-22T02:16:59.398Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1960 | 2024-08-22T02:16:59.416Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1961 | 2024-08-22T02:16:59.416Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1962 | 2024-08-22T02:16:59.437Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1963 | 2024-08-22T02:16:59.437Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1964 | 2024-08-22T02:16:59.456Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1965 | 2024-08-22T02:16:59.456Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1966 | 2024-08-22T02:16:59.456Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1967 | 2024-08-22T02:16:59.457Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 1968 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 1969 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 1970 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 1971 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 1972 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 1973 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 1974 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 1975 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 1976 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 1977 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 1978 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 1979 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 1980 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 1981 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 1982 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 1983 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 1984 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 1985 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 1986 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 1987 | 2024-08-22T02:16:59.552Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 490046
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = 5d290b50-084b-4b95-b91a-bba552ae5d1f
response_code = 500
uri = /omicron-zones
|
| 1988 | 2024-08-22T02:16:59.552Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "5d290b50-084b-4b95-b91a-bba552ae5d1f", "content-length": "124", "date": "Thu, 22 Aug 2024 02:16:59 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "5d290b50-084b-4b95-b91a-bba552ae5d1f" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 2.960886524s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1989 | 2024-08-22T02:17:00.498Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 1.403562881s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:424
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 1990 | 2024-08-22T02:17:02.523Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1991 | 2024-08-22T02:17:02.523Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1992 | 2024-08-22T02:17:02.523Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1993 | 2024-08-22T02:17:02.545Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1994 | 2024-08-22T02:17:02.545Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1995 | 2024-08-22T02:17:02.563Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1996 | 2024-08-22T02:17:02.563Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1997 | 2024-08-22T02:17:02.583Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 1998 | 2024-08-22T02:17:02.583Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1999 | 2024-08-22T02:17:02.601Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2000 | 2024-08-22T02:17:02.601Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2001 | 2024-08-22T02:17:02.619Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2002 | 2024-08-22T02:17:02.619Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2003 | 2024-08-22T02:17:02.637Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2004 | 2024-08-22T02:17:02.637Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2005 | 2024-08-22T02:17:02.656Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2006 | 2024-08-22T02:17:02.656Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2007 | 2024-08-22T02:17:02.674Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2008 | 2024-08-22T02:17:02.674Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 2009 | 2024-08-22T02:17:02.692Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2010 | 2024-08-22T02:17:02.692Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 2011 | 2024-08-22T02:17:02.710Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2012 | 2024-08-22T02:17:02.710Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 2013 | 2024-08-22T02:17:02.730Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2014 | 2024-08-22T02:17:02.730Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2015 | 2024-08-22T02:17:02.747Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2016 | 2024-08-22T02:17:02.748Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2017 | 2024-08-22T02:17:02.765Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2018 | 2024-08-22T02:17:02.765Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2019 | 2024-08-22T02:17:02.784Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2020 | 2024-08-22T02:17:02.784Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2021 | 2024-08-22T02:17:02.803Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2022 | 2024-08-22T02:17:02.803Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2023 | 2024-08-22T02:17:02.822Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2024 | 2024-08-22T02:17:02.822Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2025 | 2024-08-22T02:17:02.844Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2026 | 2024-08-22T02:17:02.844Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2027 | 2024-08-22T02:17:02.862Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2028 | 2024-08-22T02:17:02.862Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2029 | 2024-08-22T02:17:02.880Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2030 | 2024-08-22T02:17:02.881Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2031 | 2024-08-22T02:17:02.899Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2032 | 2024-08-22T02:17:02.899Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2033 | 2024-08-22T02:17:02.899Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2034 | 2024-08-22T02:17:02.899Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2035 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2036 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2037 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2038 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2039 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2040 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2041 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2042 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2043 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2044 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2045 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2046 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2047 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2048 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2049 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2050 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2051 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2052 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2053 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2054 | 2024-08-22T02:17:02.992Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 467803
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = 007128c6-f918-436c-a2b9-3c23181f6814
response_code = 500
uri = /omicron-zones
|
| 2055 | 2024-08-22T02:17:02.992Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "007128c6-f918-436c-a2b9-3c23181f6814", "content-length": "124", "date": "Thu, 22 Aug 2024 02:17:02 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "007128c6-f918-436c-a2b9-3c23181f6814" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 1.901684162s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2056 | 2024-08-22T02:17:04.900Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2057 | 2024-08-22T02:17:04.901Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2058 | 2024-08-22T02:17:04.901Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 2059 | 2024-08-22T02:17:04.924Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2060 | 2024-08-22T02:17:04.924Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 2061 | 2024-08-22T02:17:04.944Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2062 | 2024-08-22T02:17:04.944Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 2063 | 2024-08-22T02:17:04.963Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2064 | 2024-08-22T02:17:04.963Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2065 | 2024-08-22T02:17:04.984Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2066 | 2024-08-22T02:17:04.984Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2067 | 2024-08-22T02:17:05.002Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2068 | 2024-08-22T02:17:05.003Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2069 | 2024-08-22T02:17:05.022Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2070 | 2024-08-22T02:17:05.022Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2071 | 2024-08-22T02:17:05.043Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2072 | 2024-08-22T02:17:05.043Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2073 | 2024-08-22T02:17:05.063Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2074 | 2024-08-22T02:17:05.063Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 2075 | 2024-08-22T02:17:05.083Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2076 | 2024-08-22T02:17:05.083Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 2077 | 2024-08-22T02:17:05.102Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2078 | 2024-08-22T02:17:05.102Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 2079 | 2024-08-22T02:17:05.121Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2080 | 2024-08-22T02:17:05.121Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2081 | 2024-08-22T02:17:05.140Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2082 | 2024-08-22T02:17:05.140Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2083 | 2024-08-22T02:17:05.158Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2084 | 2024-08-22T02:17:05.158Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2085 | 2024-08-22T02:17:05.177Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2086 | 2024-08-22T02:17:05.177Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2087 | 2024-08-22T02:17:05.196Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2088 | 2024-08-22T02:17:05.196Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2089 | 2024-08-22T02:17:05.215Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2090 | 2024-08-22T02:17:05.215Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2091 | 2024-08-22T02:17:05.233Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2092 | 2024-08-22T02:17:05.233Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2093 | 2024-08-22T02:17:05.255Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2094 | 2024-08-22T02:17:05.255Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2095 | 2024-08-22T02:17:05.275Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2096 | 2024-08-22T02:17:05.275Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2097 | 2024-08-22T02:17:05.295Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2098 | 2024-08-22T02:17:05.295Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2099 | 2024-08-22T02:17:05.295Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2100 | 2024-08-22T02:17:05.295Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2101 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2102 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2103 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2104 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2105 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2106 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2107 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2108 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2109 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2110 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2111 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2112 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2113 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2114 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2115 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2116 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2117 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2118 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2119 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2120 | 2024-08-22T02:17:05.386Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 484302
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = 964e97ea-04e5-4841-8af6-8daad402d2b2
response_code = 500
uri = /omicron-zones
|
| 2121 | 2024-08-22T02:17:05.386Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "964e97ea-04e5-4841-8af6-8daad402d2b2", "content-length": "124", "date": "Thu, 22 Aug 2024 02:17:04 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "964e97ea-04e5-4841-8af6-8daad402d2b2" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 2.650660898s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2122 | 2024-08-22T02:17:08.046Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2123 | 2024-08-22T02:17:08.046Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2124 | 2024-08-22T02:17:08.046Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 2125 | 2024-08-22T02:17:08.069Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2126 | 2024-08-22T02:17:08.069Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 2127 | 2024-08-22T02:17:08.090Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2128 | 2024-08-22T02:17:08.090Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 2129 | 2024-08-22T02:17:08.109Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2130 | 2024-08-22T02:17:08.109Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2131 | 2024-08-22T02:17:08.129Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2132 | 2024-08-22T02:17:08.129Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2133 | 2024-08-22T02:17:08.147Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2134 | 2024-08-22T02:17:08.148Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2135 | 2024-08-22T02:17:08.166Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2136 | 2024-08-22T02:17:08.166Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2137 | 2024-08-22T02:17:08.184Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2138 | 2024-08-22T02:17:08.184Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2139 | 2024-08-22T02:17:08.203Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2140 | 2024-08-22T02:17:08.203Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 2141 | 2024-08-22T02:17:08.222Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2142 | 2024-08-22T02:17:08.222Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 2143 | 2024-08-22T02:17:08.240Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2144 | 2024-08-22T02:17:08.241Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 2145 | 2024-08-22T02:17:08.259Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2146 | 2024-08-22T02:17:08.259Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2147 | 2024-08-22T02:17:08.277Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2148 | 2024-08-22T02:17:08.277Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2149 | 2024-08-22T02:17:08.295Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2150 | 2024-08-22T02:17:08.295Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2151 | 2024-08-22T02:17:08.314Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2152 | 2024-08-22T02:17:08.314Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2153 | 2024-08-22T02:17:08.335Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2154 | 2024-08-22T02:17:08.335Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2155 | 2024-08-22T02:17:08.354Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2156 | 2024-08-22T02:17:08.354Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2157 | 2024-08-22T02:17:08.373Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2158 | 2024-08-22T02:17:08.373Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2159 | 2024-08-22T02:17:08.392Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2160 | 2024-08-22T02:17:08.392Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2161 | 2024-08-22T02:17:08.411Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2162 | 2024-08-22T02:17:08.411Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2163 | 2024-08-22T02:17:08.430Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2164 | 2024-08-22T02:17:08.430Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2165 | 2024-08-22T02:17:08.430Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2166 | 2024-08-22T02:17:08.430Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2167 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2168 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2169 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2170 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2171 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2172 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2173 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2174 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2175 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2176 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2177 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2178 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2179 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2180 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2181 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2182 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2183 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2184 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2185 | 2024-08-22T02:17:08.523Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2186 | 2024-08-22T02:17:08.523Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 476280
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = 5676d99c-7597-443c-8518-71b0a25dfba4
response_code = 500
uri = /omicron-zones
|
| 2187 | 2024-08-22T02:17:08.524Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "5676d99c-7597-443c-8518-71b0a25dfba4", "content-length": "124", "date": "Thu, 22 Aug 2024 02:17:08 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "5676d99c-7597-443c-8518-71b0a25dfba4" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 4.000145549s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2188 | 2024-08-22T02:17:12.536Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2189 | 2024-08-22T02:17:12.536Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2190 | 2024-08-22T02:17:12.536Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 2191 | 2024-08-22T02:17:12.556Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2192 | 2024-08-22T02:17:12.556Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 2193 | 2024-08-22T02:17:12.576Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2194 | 2024-08-22T02:17:12.576Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 2195 | 2024-08-22T02:17:12.597Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2196 | 2024-08-22T02:17:12.597Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2197 | 2024-08-22T02:17:12.614Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2198 | 2024-08-22T02:17:12.614Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2199 | 2024-08-22T02:17:12.632Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2200 | 2024-08-22T02:17:12.632Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2201 | 2024-08-22T02:17:12.651Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2202 | 2024-08-22T02:17:12.651Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2203 | 2024-08-22T02:17:12.670Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2204 | 2024-08-22T02:17:12.670Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2205 | 2024-08-22T02:17:12.687Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2206 | 2024-08-22T02:17:12.687Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 2207 | 2024-08-22T02:17:12.707Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2208 | 2024-08-22T02:17:12.707Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 2209 | 2024-08-22T02:17:12.725Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2210 | 2024-08-22T02:17:12.725Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 2211 | 2024-08-22T02:17:12.744Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2212 | 2024-08-22T02:17:12.744Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2213 | 2024-08-22T02:17:12.762Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2214 | 2024-08-22T02:17:12.762Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2215 | 2024-08-22T02:17:12.781Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2216 | 2024-08-22T02:17:12.781Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2217 | 2024-08-22T02:17:12.799Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2218 | 2024-08-22T02:17:12.799Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2219 | 2024-08-22T02:17:12.817Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2220 | 2024-08-22T02:17:12.817Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2221 | 2024-08-22T02:17:12.835Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2222 | 2024-08-22T02:17:12.835Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2223 | 2024-08-22T02:17:12.854Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2224 | 2024-08-22T02:17:12.854Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2225 | 2024-08-22T02:17:12.871Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2226 | 2024-08-22T02:17:12.871Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2227 | 2024-08-22T02:17:12.890Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2228 | 2024-08-22T02:17:12.890Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2229 | 2024-08-22T02:17:12.908Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2230 | 2024-08-22T02:17:12.908Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2231 | 2024-08-22T02:17:12.909Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2232 | 2024-08-22T02:17:12.909Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2233 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2234 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2235 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2236 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2237 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2238 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2239 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2240 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2241 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2242 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2243 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2244 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2245 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2246 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2247 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2248 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2249 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2250 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2251 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3185
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2252 | 2024-08-22T02:17:13.003Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959", TimeNotSynchronized), ("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c", TimeNotSynchronized), ("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d", TimeNotSynchronized), ("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf", TimeNotSynchronized), ("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46", TimeNotSynchronized), ("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7", TimeNotSynchronized), ("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906", TimeNotSynchronized), ("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3", TimeNotSynchronized), ("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f", TimeNotSynchronized), ("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f", TimeNotSynchronized), ("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2", TimeNotSynchronized), ("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33", TimeNotSynchronized), ("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df", TimeNotSynchronized), ("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4", TimeNotSynchronized), ("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864", TimeNotSynchronized), ("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7", TimeNotSynchronized), ("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49", TimeNotSynchronized), ("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576", TimeNotSynchronized), ("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902
latency_us = 465824
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = f26465bd-1a18-447c-9e30-c72f761c7280
response_code = 500
uri = /omicron-zones
|
| 2253 | 2024-08-22T02:17:13.003Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f26465bd-1a18-447c-9e30-c72f761c7280", "content-length": "124", "date": "Thu, 22 Aug 2024 02:17:12 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f26465bd-1a18-447c-9e30-c72f761c7280" }
file = sled-agent/src/rack_setup/service.rs:476
retry_after = 4.966092035s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2254 | 2024-08-22T02:17:16.981Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 3.25668983s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:424
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 2255 | 2024-08-22T02:17:17.984Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b57211b8-12b5-4756-ac2b-d3438364d75d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2662fc93-3f86-483e-b618-470614727673, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7520678c-5071-410e-9d81-975f46327871, kind: Service { id: d1bb411d-1cd9-4268-99d6-f46af4187aa1 }, name: Name("ntp-d1bb411d-1cd9-4268-99d6-f46af4187aa1"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 21])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: a3ade7be-87ab-4882-beda-b8c4b351c104, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42ef3e65-0870-4c60-8479-018e44e50e71, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 249b0525-e315-49f0-bea0-213c52bdfd49, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83b9c234-fcfa-4fe9-b4b7-c61db9754b46, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a62caba8-947d-463e-8485-54a699632f6c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 383c9379-db34-4f35-bbb7-0ff693eb5864, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 60d227cb-8f8d-4ba6-8fed-61d664284906, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External } } } }, OmicronZoneConfig { id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: d1073410-4570-439e-a9f7-a33b9380747f, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External } } } }, OmicronZoneConfig { id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External } } } }, OmicronZoneConfig { id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External } } } }, OmicronZoneConfig { id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:439
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2256 | 2024-08-22T02:17:17.985Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2257 | 2024-08-22T02:17:17.985Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 0ad4970f-3542-4f24-8bd3-6c96fb94410c, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 2258 | 2024-08-22T02:17:18.007Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2259 | 2024-08-22T02:17:18.007Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: b57211b8-12b5-4756-ac2b-d3438364d75d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 2260 | 2024-08-22T02:17:18.028Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2261 | 2024-08-22T02:17:18.028Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2662fc93-3f86-483e-b618-470614727673, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 2262 | 2024-08-22T02:17:18.049Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2263 | 2024-08-22T02:17:18.049Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 3cd84b19-998e-453f-b3bf-47ee83dcc808, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2264 | 2024-08-22T02:17:18.070Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2265 | 2024-08-22T02:17:18.070Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d533f1e0-d3b8-466c-ae7a-c4d68d1d6054, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2266 | 2024-08-22T02:17:18.088Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2267 | 2024-08-22T02:17:18.088Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a3ade7be-87ab-4882-beda-b8c4b351c104, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2268 | 2024-08-22T02:17:18.106Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2269 | 2024-08-22T02:17:18.106Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 42ef3e65-0870-4c60-8479-018e44e50e71, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2270 | 2024-08-22T02:17:18.125Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2271 | 2024-08-22T02:17:18.125Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e2f83012-ffa8-48c5-b9f8-a3531cebeca5, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 2272 | 2024-08-22T02:17:18.145Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2273 | 2024-08-22T02:17:18.145Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e3cfd6ed-225d-4c9e-a437-60c94e80df50, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 2274 | 2024-08-22T02:17:18.162Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2275 | 2024-08-22T02:17:18.162Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 2276 | 2024-08-22T02:17:18.183Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2277 | 2024-08-22T02:17:18.183Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: e604cd42-8d92-4bb5-b07a-6931fdbf2ec7, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 2278 | 2024-08-22T02:17:18.199Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2279 | 2024-08-22T02:17:18.199Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 8e7742e8-523b-430d-a69a-9f9f4d4a7adf, dataset_name: DatasetName { pool_name: ZpoolName { id: a3847786-8a2e-45ee-87a5-075f35fc7eea (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2280 | 2024-08-22T02:17:18.219Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2281 | 2024-08-22T02:17:18.219Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: a515946e-cb4e-4ebb-b0f1-eea57fa935f3, dataset_name: DatasetName { pool_name: ZpoolName { id: c613840f-0c10-41fb-9afd-d136750700f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2282 | 2024-08-22T02:17:18.238Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2283 | 2024-08-22T02:17:18.238Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: d1073410-4570-439e-a9f7-a33b9380747f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2beabcd5-0369-41ca-9d09-9a7ad5ca2181 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2284 | 2024-08-22T02:17:18.258Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2285 | 2024-08-22T02:17:18.258Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 2b4e4c46-69fb-47e0-bfb6-01d380971bb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 817bc2ac-fa69-4e9b-8fd6-6f77d7295e99 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2286 | 2024-08-22T02:17:18.277Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2287 | 2024-08-22T02:17:18.277Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 558fb2d9-3acc-4a64-bff5-f85a077f1e33, dataset_name: DatasetName { pool_name: ZpoolName { id: 50197822-d2db-4c18-aae1-d7d7bdd76af4 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2288 | 2024-08-22T02:17:18.295Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2289 | 2024-08-22T02:17:18.295Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: dce94a66-3e8e-4ef2-a278-14fcbaefb3df, dataset_name: DatasetName { pool_name: ZpoolName { id: 39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2290 | 2024-08-22T02:17:18.313Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2291 | 2024-08-22T02:17:18.313Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: fc052a6b-ed5e-4aa6-b998-97446b9d35e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 93a0a2e2-5fff-4e15-9016-1c60e061297f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2292 | 2024-08-22T02:17:18.331Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2293 | 2024-08-22T02:17:18.331Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: bf20743c-2c90-4d79-8cba-0a0d296d31a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 01770c46-d16d-4f08-bf5d-3bc7b219d504 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2294 | 2024-08-22T02:17:18.349Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:406
|
| 2295 | 2024-08-22T02:17:18.349Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:773
request = NewFilesystemRequest { dataset_id: 7e931c9a-fc76-4509-b7db-fa4aa9e03576, dataset_name: DatasetName { pool_name: ZpoolName { id: 555e0f8f-2230-434a-8ce6-0ed459a335cf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 2296 | 2024-08-22T02:17:18.368Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2297 | 2024-08-22T02:17:18.368Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2298 | 2024-08-22T02:17:18.368Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2299 | 2024-08-22T02:17:18.368Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2300 | 2024-08-22T02:17:18.454Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:260
|
| 2301 | 2024-08-22T02:17:18.456Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:260
|
| 2302 | 2024-08-22T02:17:18.501Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:260
|
| 2303 | 2024-08-22T02:17:18.501Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2304 | 2024-08-22T02:17:18.501Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2305 | 2024-08-22T02:17:18.501Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2306 | 2024-08-22T02:17:18.503Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:260
|
| 2307 | 2024-08-22T02:17:18.525Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:260
|
| 2308 | 2024-08-22T02:17:18.526Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2309 | 2024-08-22T02:17:18.538Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2310 | 2024-08-22T02:17:18.538Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2311 | 2024-08-22T02:17:18.712Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2312 | 2024-08-22T02:17:18.712Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2313 | 2024-08-22T02:17:18.712Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2314 | 2024-08-22T02:17:18.712Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2315 | 2024-08-22T02:17:18.712Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2316 | 2024-08-22T02:17:18.712Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2317 | 2024-08-22T02:17:18.712Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2318 | 2024-08-22T02:17:18.712Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2319 | 2024-08-22T02:17:18.713Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
file = illumos-utils/src/zone.rs:316
|
| 2320 | 2024-08-22T02:17:18.719Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
file = illumos-utils/src/zone.rs:316
|
| 2321 | 2024-08-22T02:17:18.724Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
file = illumos-utils/src/zone.rs:316
|
| 2322 | 2024-08-22T02:17:18.730Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
file = illumos-utils/src/zone.rs:316
|
| 2323 | 2024-08-22T02:17:18.735Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
file = illumos-utils/src/zone.rs:316
|
| 2324 | 2024-08-22T02:17:18.740Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
file = illumos-utils/src/zone.rs:316
|
| 2325 | 2024-08-22T02:17:18.897Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
file = illumos-utils/src/zone.rs:354
|
| 2326 | 2024-08-22T02:17:18.903Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
file = illumos-utils/src/zone.rs:354
|
| 2327 | 2024-08-22T02:17:18.907Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
file = illumos-utils/src/zone.rs:354
|
| 2328 | 2024-08-22T02:17:18.913Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
file = illumos-utils/src/zone.rs:354
|
| 2329 | 2024-08-22T02:17:18.952Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:277
|
| 2330 | 2024-08-22T02:17:18.952Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
file = illumos-utils/src/zone.rs:354
|
| 2331 | 2024-08-22T02:17:18.956Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
file = illumos-utils/src/zone.rs:354
|
| 2332 | 2024-08-22T02:17:18.961Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
file = illumos-utils/src/zone.rs:316
|
| 2333 | 2024-08-22T02:17:18.966Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
file = illumos-utils/src/zone.rs:316
|
| 2334 | 2024-08-22T02:17:18.971Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
file = illumos-utils/src/zone.rs:316
|
| 2335 | 2024-08-22T02:17:18.976Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
file = illumos-utils/src/zone.rs:316
|
| 2336 | 2024-08-22T02:17:18.981Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
file = illumos-utils/src/zone.rs:316
|
| 2337 | 2024-08-22T02:17:18.986Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
file = illumos-utils/src/zone.rs:316
|
| 2338 | 2024-08-22T02:17:18.992Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
file = illumos-utils/src/zone.rs:316
|
| 2339 | 2024-08-22T02:17:18.997Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
file = illumos-utils/src/zone.rs:316
|
| 2340 | 2024-08-22T02:17:19.039Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:277
|
| 2341 | 2024-08-22T02:17:19.071Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:277
|
| 2342 | 2024-08-22T02:17:19.071Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
file = illumos-utils/src/zone.rs:354
|
| 2343 | 2024-08-22T02:17:19.077Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
file = illumos-utils/src/zone.rs:354
|
| 2344 | 2024-08-22T02:17:19.083Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
file = illumos-utils/src/zone.rs:354
|
| 2345 | 2024-08-22T02:17:19.089Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
file = illumos-utils/src/zone.rs:354
|
| 2346 | 2024-08-22T02:17:19.094Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
file = illumos-utils/src/zone.rs:354
|
| 2347 | 2024-08-22T02:17:19.100Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
file = illumos-utils/src/zone.rs:354
|
| 2348 | 2024-08-22T02:17:19.106Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
file = illumos-utils/src/zone.rs:354
|
| 2349 | 2024-08-22T02:17:19.112Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
file = illumos-utils/src/zone.rs:354
|
| 2350 | 2024-08-22T02:17:19.183Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:277
|
| 2351 | 2024-08-22T02:17:19.239Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:277
|
| 2352 | 2024-08-22T02:17:19.245Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:302
|
| 2353 | 2024-08-22T02:17:19.245Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:194
|
| 2354 | 2024-08-22T02:17:19.292Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:424
port = Port { inner: PortInner(PortData { name: "opte1", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 250, 221]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }
|
| 2355 | 2024-08-22T02:17:19.292Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1235
zone_type = external_dns
|
| 2356 | 2024-08-22T02:17:19.294Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:302
|
| 2357 | 2024-08-22T02:17:19.294Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:194
|
| 2358 | 2024-08-22T02:17:19.338Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:424
port = Port { inner: PortInner(PortData { name: "opte2", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 156, 98]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }
|
| 2359 | 2024-08-22T02:17:19.338Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1235
zone_type = external_dns
|
| 2360 | 2024-08-22T02:17:19.340Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:302
|
| 2361 | 2024-08-22T02:17:19.340Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:194
|
| 2362 | 2024-08-22T02:17:19.387Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:424
port = Port { inner: PortInner(PortData { name: "opte3", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 180, 42]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }
|
| 2363 | 2024-08-22T02:17:19.387Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1235
zone_type = nexus
|
| 2364 | 2024-08-22T02:17:19.392Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:302
|
| 2365 | 2024-08-22T02:17:19.392Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:194
|
| 2366 | 2024-08-22T02:17:19.552Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:424
port = Port { inner: PortInner(PortData { name: "opte4", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 255, 202]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }
|
| 2367 | 2024-08-22T02:17:19.552Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1235
zone_type = nexus
|
| 2368 | 2024-08-22T02:17:19.553Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:302
|
| 2369 | 2024-08-22T02:17:19.553Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:194
|
| 2370 | 2024-08-22T02:17:19.613Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:424
port = Port { inner: PortInner(PortData { name: "opte5", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 216, 61]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }
|
| 2371 | 2024-08-22T02:17:19.613Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1235
zone_type = nexus
|
| 2372 | 2024-08-22T02:17:19.614Z | INFO | SledAgent (ServiceManager): no nat entry found for: 10.151.2.167
file = clients/dpd-client/src/lib.rs:83
|
| 2373 | 2024-08-22T02:17:19.614Z | INFO | SledAgent (ServiceManager): creating nat entry for: 10.151.2.167
file = clients/dpd-client/src/lib.rs:90
|
| 2374 | 2024-08-22T02:17:19.614Z | INFO | SledAgent (ServiceManager): no nat entry found for: 10.151.2.165
file = clients/dpd-client/src/lib.rs:83
|
| 2375 | 2024-08-22T02:17:19.614Z | INFO | SledAgent (ServiceManager): creating nat entry for: 10.151.2.165
file = clients/dpd-client/src/lib.rs:90
|
| 2376 | 2024-08-22T02:17:19.614Z | INFO | SledAgent (ServiceManager): no nat entry found for: 10.151.2.164
file = clients/dpd-client/src/lib.rs:83
|
| 2377 | 2024-08-22T02:17:19.614Z | INFO | SledAgent (ServiceManager): creating nat entry for: 10.151.2.164
file = clients/dpd-client/src/lib.rs:90
|
| 2378 | 2024-08-22T02:17:19.614Z | INFO | SledAgent (ServiceManager): no nat entry found for: 10.151.2.166
file = clients/dpd-client/src/lib.rs:83
|
| 2379 | 2024-08-22T02:17:19.614Z | INFO | SledAgent (ServiceManager): creating nat entry for: 10.151.2.166
file = clients/dpd-client/src/lib.rs:90
|
| 2380 | 2024-08-22T02:17:19.617Z | INFO | SledAgent (ServiceManager): no nat entry found for: 10.151.2.168
file = clients/dpd-client/src/lib.rs:83
|
| 2381 | 2024-08-22T02:17:19.617Z | INFO | SledAgent (ServiceManager): creating nat entry for: 10.151.2.168
file = clients/dpd-client/src/lib.rs:90
|
| 2382 | 2024-08-22T02:17:19.617Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: 10.151.2.167
file = clients/dpd-client/src/lib.rs:118
|
| 2383 | 2024-08-22T02:17:19.618Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: 10.151.2.164
file = clients/dpd-client/src/lib.rs:118
|
| 2384 | 2024-08-22T02:17:19.620Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: 10.151.2.166
file = clients/dpd-client/src/lib.rs:118
|
| 2385 | 2024-08-22T02:17:19.620Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2386 | 2024-08-22T02:17:19.620Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2387 | 2024-08-22T02:17:19.622Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: 10.151.2.165
file = clients/dpd-client/src/lib.rs:118
|
| 2388 | 2024-08-22T02:17:19.625Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2389 | 2024-08-22T02:17:19.625Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2390 | 2024-08-22T02:17:19.625Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: 10.151.2.168
file = clients/dpd-client/src/lib.rs:118
|
| 2391 | 2024-08-22T02:17:19.753Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2392 | 2024-08-22T02:17:19.753Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
file = illumos-utils/src/zone.rs:316
|
| 2393 | 2024-08-22T02:17:19.762Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
file = illumos-utils/src/zone.rs:316
|
| 2394 | 2024-08-22T02:17:19.768Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
file = illumos-utils/src/zone.rs:316
|
| 2395 | 2024-08-22T02:17:19.774Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
file = illumos-utils/src/zone.rs:316
|
| 2396 | 2024-08-22T02:17:19.814Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
file = illumos-utils/src/zone.rs:354
|
| 2397 | 2024-08-22T02:17:19.820Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
file = illumos-utils/src/zone.rs:354
|
| 2398 | 2024-08-22T02:17:19.827Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
file = illumos-utils/src/zone.rs:354
|
| 2399 | 2024-08-22T02:17:19.833Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
file = illumos-utils/src/zone.rs:354
|
| 2400 | 2024-08-22T02:17:19.839Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
file = illumos-utils/src/zone.rs:316
|
| 2401 | 2024-08-22T02:17:19.869Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
file = illumos-utils/src/zone.rs:354
|
| 2402 | 2024-08-22T02:17:21.296Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService14'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::f'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::f'/>
<propval type="astring" name="listen_port" value='17000'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2403 | 2024-08-22T02:17:21.296Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2404 | 2024-08-22T02:17:21.382Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService15'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::14'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_2beabcd5-0369-41ca-9d09-9a7ad5ca2181/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::14'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='e9076dfa-3f98-4c92-aebe-a722e53c36f7'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2405 | 2024-08-22T02:17:21.382Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2406 | 2024-08-22T02:17:21.391Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService10'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::15'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_817bc2ac-fa69-4e9b-8fd6-6f77d7295e99/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::15'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='691f20da-1279-4964-ae28-0107436a38cc'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2407 | 2024-08-22T02:17:21.391Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService11'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::1a'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_555e0f8f-2230-434a-8ce6-0ed459a335cf/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::1a'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='c2996114-cec8-4f3d-b762-bfb3fc0a2e5e'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2408 | 2024-08-22T02:17:21.392Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2409 | 2024-08-22T02:17:21.399Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2410 | 2024-08-22T02:17:21.426Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService16'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::17'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::17'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='0a40c617-e8c2-4448-be5e-897ccd50a930'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2411 | 2024-08-22T02:17:21.427Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2412 | 2024-08-22T02:17:21.480Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService12'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::19'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_01770c46-d16d-4f08-bf5d-3bc7b219d504/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::19'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='c130bb53-0aae-4b73-9cba-a9d63bca8cae'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2413 | 2024-08-22T02:17:21.480Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2414 | 2024-08-22T02:17:21.506Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:52340
|
| 2415 | 2024-08-22T02:17:21.506Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 164
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:52340
req_id = e874ef28-187f-40e5-a6fd-3d473fbbb628
response_code = 200
uri = /baseboard
|
| 2416 | 2024-08-22T02:17:23.367Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 25.065969ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2417 | 2024-08-22T02:17:23.407Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 61.78172ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2418 | 2024-08-22T02:17:23.496Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 51.358964ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2419 | 2024-08-22T02:17:23.578Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 71.955216ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2420 | 2024-08-22T02:17:30.321Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 26.907792ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2421 | 2024-08-22T02:17:30.401Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 64.602287ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2422 | 2024-08-22T02:17:30.511Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 57.216981ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2423 | 2024-08-22T02:17:30.658Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 110.602641ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2424 | 2024-08-22T02:17:30.738Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 129.159467ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2425 | 2024-08-22T02:17:30.870Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 105.91963ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2426 | 2024-08-22T02:17:31.003Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 105.735579ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2427 | 2024-08-22T02:17:31.103Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 99.953135ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2428 | 2024-08-22T02:17:31.146Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService22'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::18'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_93a0a2e2-5fff-4e15-9016-1c60e061297f/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::18'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='ddb6ba58-138c-43c5-a0bf-c93ffe267b69'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2429 | 2024-08-22T02:17:31.146Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService23'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::16'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_50197822-d2db-4c18-aae1-d7d7bdd76af4/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::16'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='2bedd77b-9456-4736-9e1b-1553969802c2'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2430 | 2024-08-22T02:17:31.284Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 148.92579ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2431 | 2024-08-22T02:17:31.445Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 282.022397ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2432 | 2024-08-22T02:17:31.618Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 281.817131ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2433 | 2024-08-22T02:17:31.688Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 217.345805ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2434 | 2024-08-22T02:17:31.963Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 144.401196ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2435 | 2024-08-22T02:17:32.002Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 154.177795ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2436 | 2024-08-22T02:17:32.004Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService17'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::12'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_a3847786-8a2e-45ee-87a5-075f35fc7eea/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::12'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='21fa94c3-4438-4d88-a4da-8066087faa61'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2437 | 2024-08-22T02:17:32.005Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService19'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::10'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::10'/>
<propval type="astring" name="listen_port" value='17000'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2438 | 2024-08-22T02:17:32.006Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService20'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::11'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::11'/>
<propval type="astring" name="listen_port" value='17000'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2439 | 2024-08-22T02:17:32.006Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2440 | 2024-08-22T02:17:32.013Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2441 | 2024-08-22T02:17:32.036Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 296.455332ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2442 | 2024-08-22T02:17:32.049Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 494.817341ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2443 | 2024-08-22T02:17:32.062Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 403.753471ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2444 | 2024-08-22T02:17:32.072Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 424.779353ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2445 | 2024-08-22T02:17:32.072Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2446 | 2024-08-22T02:17:32.077Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2447 | 2024-08-22T02:17:32.083Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2448 | 2024-08-22T02:17:32.405Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 570.259184ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2449 | 2024-08-22T02:17:32.462Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 283.123164ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2450 | 2024-08-22T02:17:32.491Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 559.207149ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2451 | 2024-08-22T02:17:32.520Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 866.297894ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2452 | 2024-08-22T02:17:32.599Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 943.642686ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2453 | 2024-08-22T02:17:32.619Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 1.065104453s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2454 | 2024-08-22T02:17:32.744Z | INFO | SledAgent (ServiceManager): Profile for oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService18'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::d'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/oximeter">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="id" value='83b9c234-fcfa-4fe9-b4b7-c61db9754b46'/>
<propval type="astring" name="address" value='[fd00:1122:3344:101::d]:12223'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2455 | 2024-08-22T02:17:32.744Z | INFO | SledAgent (ServiceManager): Booting oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2456 | 2024-08-22T02:17:32.769Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 760.836638ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2457 | 2024-08-22T02:17:32.847Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 67.124421ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2458 | 2024-08-22T02:17:32.900Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 59.659801ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2459 | 2024-08-22T02:17:32.917Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 27.70278ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2460 | 2024-08-22T02:17:32.932Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 136.697352ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2461 | 2024-08-22T02:17:32.933Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService21'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::13'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_c613840f-0c10-41fb-9afd-d136750700f5/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::13'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='5af87f3a-7820-4971-9bdd-8abfddba7069'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2462 | 2024-08-22T02:17:32.946Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 39.125729ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2463 | 2024-08-22T02:17:32.946Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2464 | 2024-08-22T02:17:32.967Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 124.218371ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2465 | 2024-08-22T02:17:32.980Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 77.50501ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2466 | 2024-08-22T02:17:32.993Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 650.908726ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2467 | 2024-08-22T02:17:33.005Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 106.496635ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2468 | 2024-08-22T02:17:33.041Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 52.403693ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2469 | 2024-08-22T02:17:33.095Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 1.103649087s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2470 | 2024-08-22T02:17:33.197Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 212.469876ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2471 | 2024-08-22T02:17:33.218Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 165.443451ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2472 | 2024-08-22T02:17:33.240Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 127.357579ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2473 | 2024-08-22T02:17:33.261Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 163.756072ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2474 | 2024-08-22T02:17:33.286Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 106.715667ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2475 | 2024-08-22T02:17:33.388Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 57.287192ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2476 | 2024-08-22T02:17:33.418Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 272.595153ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2477 | 2024-08-22T02:17:33.439Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 206.312586ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2478 | 2024-08-22T02:17:33.458Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 969.045914ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2479 | 2024-08-22T02:17:33.496Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 569.586972ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2480 | 2024-08-22T02:17:33.515Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 440.206316ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2481 | 2024-08-22T02:17:33.533Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 225.733862ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2482 | 2024-08-22T02:17:33.549Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 46.677976ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2483 | 2024-08-22T02:17:33.571Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 710.91562ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2484 | 2024-08-22T02:17:33.600Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 1.180750474s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2485 | 2024-08-22T02:17:33.634Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 54.144685ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2486 | 2024-08-22T02:17:33.663Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 62.679151ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2487 | 2024-08-22T02:17:33.685Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 1.114655136s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2488 | 2024-08-22T02:17:33.707Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 1.338257818s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2489 | 2024-08-22T02:17:33.729Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 1.429087814s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2490 | 2024-08-22T02:17:33.756Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 244.888861ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2491 | 2024-08-22T02:17:33.781Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 286.698617ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2492 | 2024-08-22T02:17:33.798Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 1.08129936s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2493 | 2024-08-22T02:17:33.813Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 407.305748ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2494 | 2024-08-22T02:17:33.975Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 540.633408ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2495 | 2024-08-22T02:17:34.018Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 451.472265ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2496 | 2024-08-22T02:17:34.085Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 670.510437ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2497 | 2024-08-22T02:17:34.101Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 560.364761ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2498 | 2024-08-22T02:17:34.216Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 749.632759ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2499 | 2024-08-22T02:17:34.237Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 1.163981181s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2500 | 2024-08-22T02:17:34.300Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 677.945015ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2501 | 2024-08-22T02:17:34.446Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 565.362494ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2502 | 2024-08-22T02:17:34.485Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 643.851403ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2503 | 2024-08-22T02:17:34.513Z | INFO | SledAgent (ServiceManager): Profile for oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService27'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::9'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value='opte1'/>
<propval type="astring" name="gateway" value='172.30.1.1'/>
<propval type="astring" name="ip" value='172.30.1.6'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/external_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::9]:5353'/>
<propval type="astring" name="dns_address" value='172.30.1.6:53'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2504 | 2024-08-22T02:17:34.513Z | INFO | SledAgent (ServiceManager): Booting oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2505 | 2024-08-22T02:17:34.530Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 1.013198932s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2506 | 2024-08-22T02:17:34.566Z | INFO | SledAgent (ServiceManager): Profile for oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService25'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::8'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value='opte2'/>
<propval type="astring" name="gateway" value='172.30.1.1'/>
<propval type="astring" name="ip" value='172.30.1.5'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/external_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::8]:5353'/>
<propval type="astring" name="dns_address" value='172.30.1.5:53'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2507 | 2024-08-22T02:17:34.566Z | INFO | SledAgent (ServiceManager): Booting oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2508 | 2024-08-22T02:17:34.675Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 721.33036ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2509 | 2024-08-22T02:17:34.769Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 607.417253ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2510 | 2024-08-22T02:17:34.794Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 849.879234ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2511 | 2024-08-22T02:17:34.812Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 1.257132245s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2512 | 2024-08-22T02:17:34.820Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 25.094828ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2513 | 2024-08-22T02:17:34.855Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 145.279423ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2514 | 2024-08-22T02:17:34.879Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 71.598657ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2515 | 2024-08-22T02:17:34.892Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 1.087830049s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2516 | 2024-08-22T02:17:34.962Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 136.245086ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2517 | 2024-08-22T02:17:34.979Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 975.839532ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2518 | 2024-08-22T02:17:34.992Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 1.004104351s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2519 | 2024-08-22T02:17:35.012Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 218.282322ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2520 | 2024-08-22T02:17:35.021Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 1.140654966s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2521 | 2024-08-22T02:17:35.057Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 556.91318ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2522 | 2024-08-22T02:17:35.113Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 223.652718ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2523 | 2024-08-22T02:17:35.143Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 560.960145ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2524 | 2024-08-22T02:17:35.170Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 1.420714405s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2525 | 2024-08-22T02:17:35.243Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 312.918114ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2526 | 2024-08-22T02:17:35.699Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 7.036196652s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:424
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 2527 | 2024-08-22T02:17:35.762Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 614.880281ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2528 | 2024-08-22T02:17:35.773Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 420.822199ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2529 | 2024-08-22T02:17:35.784Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 1.122557775s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2530 | 2024-08-22T02:17:35.848Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 901.179089ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2531 | 2024-08-22T02:17:35.859Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 965.856606ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2532 | 2024-08-22T02:17:35.869Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 553.746639ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2533 | 2024-08-22T02:17:35.877Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 1.063882783s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2534 | 2024-08-22T02:17:35.886Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 1.087039532s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2535 | 2024-08-22T02:17:35.894Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 1.19928774s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2536 | 2024-08-22T02:17:35.969Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 950.512199ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2537 | 2024-08-22T02:17:35.994Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 1.38770564s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2538 | 2024-08-22T02:17:36.009Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 688.39399ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2539 | 2024-08-22T02:17:36.083Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 904.950738ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2540 | 2024-08-22T02:17:36.175Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 1.362947567s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2541 | 2024-08-22T02:17:36.207Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 736.499185ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2542 | 2024-08-22T02:17:36.391Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 1.108095673s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2543 | 2024-08-22T02:17:36.435Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 868.066976ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2544 | 2024-08-22T02:17:36.604Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 544.190614ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2545 | 2024-08-22T02:17:36.711Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 1.494777611s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2546 | 2024-08-22T02:17:36.763Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 637.022857ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2547 | 2024-08-22T02:17:36.838Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 1.166437472s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2548 | 2024-08-22T02:17:36.991Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 1.263595311s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2549 | 2024-08-22T02:17:37.136Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 1.404867111s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2550 | 2024-08-22T02:17:37.148Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 794.094525ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2551 | 2024-08-22T02:17:37.160Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 1.236380519s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2552 | 2024-08-22T02:17:37.171Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 1.447026298s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2553 | 2024-08-22T02:17:37.183Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 677.445775ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2554 | 2024-08-22T02:17:37.194Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 1.06768808s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2555 | 2024-08-22T02:17:37.205Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 1.344711349s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2556 | 2024-08-22T02:17:37.206Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService24'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::b'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value='opte5'/>
<propval type="astring" name="gateway" value='172.30.2.1'/>
<propval type="astring" name="ip" value='172.30.2.6'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2557 | 2024-08-22T02:17:37.206Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService26'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::a'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value='opte3'/>
<propval type="astring" name="gateway" value='172.30.2.1'/>
<propval type="astring" name="ip" value='172.30.2.5'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2558 | 2024-08-22T02:17:37.206Z | INFO | SledAgent (ServiceManager): Booting oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2559 | 2024-08-22T02:17:37.212Z | INFO | SledAgent (ServiceManager): Booting oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2560 | 2024-08-22T02:17:37.317Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 1.252223459s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2561 | 2024-08-22T02:17:37.490Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 969.988592ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2562 | 2024-08-22T02:17:37.507Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 1.237922801s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2563 | 2024-08-22T02:17:37.521Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 1.328344294s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2564 | 2024-08-22T02:17:37.540Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 66.595182ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2565 | 2024-08-22T02:17:37.622Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 115.17433ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2566 | 2024-08-22T02:17:37.634Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 1.046599102s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2567 | 2024-08-22T02:17:37.642Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 46.722733ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2568 | 2024-08-22T02:17:37.704Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 126.72398ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2569 | 2024-08-22T02:17:37.747Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 167.974618ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2570 | 2024-08-22T02:17:37.846Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 279.270047ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2571 | 2024-08-22T02:17:37.873Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 1.344311152s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2572 | 2024-08-22T02:17:37.931Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 498.758382ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2573 | 2024-08-22T02:17:37.956Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 1.074807302s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2574 | 2024-08-22T02:17:38.017Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 891.545919ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2575 | 2024-08-22T02:17:38.140Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 218.809022ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2576 | 2024-08-22T02:17:38.219Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 865.711894ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2577 | 2024-08-22T02:17:38.269Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 832.68672ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2578 | 2024-08-22T02:17:38.280Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 1.233053951s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2579 | 2024-08-22T02:17:38.372Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 628.525347ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2580 | 2024-08-22T02:17:38.574Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 1.09679464s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2581 | 2024-08-22T02:17:38.586Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 715.557284ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2582 | 2024-08-22T02:17:38.599Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 1.130703236s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2583 | 2024-08-22T02:17:38.611Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 961.756293ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2584 | 2024-08-22T02:17:38.621Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 852.895229ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2585 | 2024-08-22T02:17:38.632Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 749.203488ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2586 | 2024-08-22T02:17:38.642Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 1.019871017s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2587 | 2024-08-22T02:17:38.693Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 1.114174433s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2588 | 2024-08-22T02:17:38.756Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 1.11615303s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2589 | 2024-08-22T02:17:38.862Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 1.429909733s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2590 | 2024-08-22T02:17:38.919Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 1.325793163s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2591 | 2024-08-22T02:17:39.011Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 518.979445ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2592 | 2024-08-22T02:17:39.039Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 1.138938958s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2593 | 2024-08-22T02:17:39.095Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 942.907072ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2594 | 2024-08-22T02:17:39.111Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 507.733147ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2595 | 2024-08-22T02:17:39.226Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 992.173679ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2596 | 2024-08-22T02:17:39.311Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 1.003337619s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2597 | 2024-08-22T02:17:39.391Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 873.250588ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2598 | 2024-08-22T02:17:39.488Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 682.843594ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2599 | 2024-08-22T02:17:39.528Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 1.17747766s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2600 | 2024-08-22T02:17:39.542Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 1.077030913s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2601 | 2024-08-22T02:17:39.586Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 933.832396ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2602 | 2024-08-22T02:17:39.631Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 1.401749809s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2603 | 2024-08-22T02:17:39.674Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 1.468463928s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2604 | 2024-08-22T02:17:39.685Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 1.409277288s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2605 | 2024-08-22T02:17:39.742Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 824.167415ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2606 | 2024-08-22T02:17:39.820Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 1.157197372s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2607 | 2024-08-22T02:17:39.884Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 917.114006ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2608 | 2024-08-22T02:17:40.046Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 730.042792ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2609 | 2024-08-22T02:17:40.101Z | INFO | SledAgent (ServiceManager): Profile for oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService13'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::e'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/clickhouse">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::e'/>
<propval type="astring" name="listen_port" value='8123'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/clickhouse-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="clickhouse_address" value='[fd00:1122:3344:101::e]:8123'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::e]:8888'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2610 | 2024-08-22T02:17:40.101Z | INFO | SledAgent (ServiceManager): Booting oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2611 | 2024-08-22T02:17:40.180Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 1.404181503s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2612 | 2024-08-22T02:17:40.188Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 854.863319ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2613 | 2024-08-22T02:17:40.226Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 590.062348ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2614 | 2024-08-22T02:17:40.254Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 1.176884701s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2615 | 2024-08-22T02:17:40.272Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 980.460759ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2616 | 2024-08-22T02:17:40.301Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 1.111107471s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2617 | 2024-08-22T02:17:40.322Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 528.17704ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2618 | 2024-08-22T02:17:40.396Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 71.293487ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2619 | 2024-08-22T02:17:40.475Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 127.331609ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2620 | 2024-08-22T02:17:40.527Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 1.419540554s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2621 | 2024-08-22T02:17:40.575Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 835.416686ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2622 | 2024-08-22T02:17:40.614Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 113.472229ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2623 | 2024-08-22T02:17:40.629Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 1.315994568s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2624 | 2024-08-22T02:17:40.717Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 673.005849ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2625 | 2024-08-22T02:17:40.739Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 466.977429ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2626 | 2024-08-22T02:17:40.860Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 1.013108652s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2627 | 2024-08-22T02:17:40.921Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 714.25544ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2628 | 2024-08-22T02:17:40.933Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 937.029493ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2629 | 2024-08-22T02:17:40.944Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 581.896593ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2630 | 2024-08-22T02:17:41.006Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 721.302166ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2631 | 2024-08-22T02:17:41.052Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 1.402809027s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2632 | 2024-08-22T02:17:41.060Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 583.988143ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2633 | 2024-08-22T02:17:41.123Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 659.500494ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2634 | 2024-08-22T02:17:41.193Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 863.032731ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2635 | 2024-08-22T02:17:41.214Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 404.791661ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2636 | 2024-08-22T02:17:41.262Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 1.347974002s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2637 | 2024-08-22T02:17:41.399Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 510.067452ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2638 | 2024-08-22T02:17:41.479Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 839.340667ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2639 | 2024-08-22T02:17:41.487Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 896.365281ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2640 | 2024-08-22T02:17:41.494Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 579.115971ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2641 | 2024-08-22T02:17:41.534Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 591.581852ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2642 | 2024-08-22T02:17:41.593Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 1.144490612s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2643 | 2024-08-22T02:17:41.628Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 691.805852ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2644 | 2024-08-22T02:17:41.645Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 1.493182045s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2645 | 2024-08-22T02:17:41.653Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 733.260003ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2646 | 2024-08-22T02:17:41.737Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 1.177610308s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2647 | 2024-08-22T02:17:41.791Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 1.406790177s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2648 | 2024-08-22T02:17:41.879Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 635.302002ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2649 | 2024-08-22T02:17:41.888Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 515.358904ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2650 | 2024-08-22T02:17:41.919Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 1.435810656s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2651 | 2024-08-22T02:17:41.954Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 530.951495ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2652 | 2024-08-22T02:17:41.963Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 1.01272576s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2653 | 2024-08-22T02:17:42.066Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 1.396830663s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2654 | 2024-08-22T02:17:42.080Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 1.476527531s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2655 | 2024-08-22T02:17:42.136Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 1.034017565s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2656 | 2024-08-22T02:17:42.333Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 696.162912ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2657 | 2024-08-22T02:17:42.347Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 518.774563ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2658 | 2024-08-22T02:17:42.394Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 1.267574851s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2659 | 2024-08-22T02:17:42.402Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 1.345029707s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2660 | 2024-08-22T02:17:42.414Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 737.904206ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2661 | 2024-08-22T02:17:42.466Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 705.832387ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2662 | 2024-08-22T02:17:42.496Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 1.387790786s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2663 | 2024-08-22T02:17:42.525Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 907.393546ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2664 | 2024-08-22T02:17:42.619Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 1.217685299s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2665 | 2024-08-22T02:17:42.726Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService28'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::c'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value='opte4'/>
<propval type="astring" name="gateway" value='172.30.2.1'/>
<propval type="astring" name="ip" value='172.30.2.7'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2666 | 2024-08-22T02:17:42.727Z | INFO | SledAgent (ServiceManager): Booting oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2667 | 2024-08-22T02:17:43.349Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 65.941493ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2668 | 2024-08-22T02:17:43.356Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 1.001565284s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2669 | 2024-08-22T02:17:43.363Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 989.364106ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2670 | 2024-08-22T02:17:43.371Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 1.175945672s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2671 | 2024-08-22T02:17:43.379Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 1.139234289s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2672 | 2024-08-22T02:17:43.386Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 788.203317ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2673 | 2024-08-22T02:17:43.394Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 757.402292ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2674 | 2024-08-22T02:17:43.404Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 998.45492ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2675 | 2024-08-22T02:17:43.415Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 1.203585863s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2676 | 2024-08-22T02:17:43.424Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 1.469284149s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2677 | 2024-08-22T02:17:43.432Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 917.62457ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2678 | 2024-08-22T02:17:43.441Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 1.487538913s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2679 | 2024-08-22T02:17:43.448Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 56.947071ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2680 | 2024-08-22T02:17:43.455Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 1.049503674s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2681 | 2024-08-22T02:17:43.471Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 1.19926497s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2682 | 2024-08-22T02:17:43.518Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 171.938003ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2683 | 2024-08-22T02:17:43.567Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 1.440939703s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2684 | 2024-08-22T02:17:43.671Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 628.134812ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2685 | 2024-08-22T02:17:43.699Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 207.922742ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2686 | 2024-08-22T02:17:43.758Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 1.06272765s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2687 | 2024-08-22T02:17:43.848Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 1.44535786s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2688 | 2024-08-22T02:17:43.894Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 1.241084884s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2689 | 2024-08-22T02:17:43.917Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 1.091482832s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2690 | 2024-08-22T02:17:44.161Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 1.351439527s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2691 | 2024-08-22T02:17:44.183Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 1.333293015s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2692 | 2024-08-22T02:17:44.308Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 879.754929ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2693 | 2024-08-22T02:17:44.359Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 506.154722ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2694 | 2024-08-22T02:17:44.367Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 1.322051056s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2695 | 2024-08-22T02:17:44.374Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 808.640428ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2696 | 2024-08-22T02:17:44.412Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 1.017536134s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2697 | 2024-08-22T02:17:44.514Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 1.123530399s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2698 | 2024-08-22T02:17:44.526Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 1.316246382s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2699 | 2024-08-22T02:17:44.556Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 1.02107496s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2700 | 2024-08-22T02:17:44.627Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 1.144511868s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2701 | 2024-08-22T02:17:44.680Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 1.384526729s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2702 | 2024-08-22T02:17:44.831Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 1.037388502s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2703 | 2024-08-22T02:17:44.875Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 617.919318ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2704 | 2024-08-22T02:17:44.903Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 830.108034ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2705 | 2024-08-22T02:17:44.938Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 986.861133ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2706 | 2024-08-22T02:17:45.017Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 916.984556ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2707 | 2024-08-22T02:17:45.026Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 1.2017099s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2708 | 2024-08-22T02:17:45.146Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 1.376339718s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2709 | 2024-08-22T02:17:45.193Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 596.341441ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2710 | 2024-08-22T02:17:45.201Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 520.419762ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2711 | 2024-08-22T02:17:45.306Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 1.309601858s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2712 | 2024-08-22T02:17:45.440Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 668.018528ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2713 | 2024-08-22T02:17:45.503Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 1.471979109s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2714 | 2024-08-22T02:17:45.523Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 1.024772275s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2715 | 2024-08-22T02:17:45.533Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 517.941189ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2716 | 2024-08-22T02:17:45.589Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 598.359107ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2717 | 2024-08-22T02:17:45.649Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 549.364609ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2718 | 2024-08-22T02:17:45.700Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 1.350245186s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2719 | 2024-08-22T02:17:45.733Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 850.038376ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2720 | 2024-08-22T02:17:45.742Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 729.047433ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2721 | 2024-08-22T02:17:45.784Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 1.195039715s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2722 | 2024-08-22T02:17:45.822Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 1.308866784s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2723 | 2024-08-22T02:17:45.865Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 1.092494708s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2724 | 2024-08-22T02:17:45.882Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 512.551661ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2725 | 2024-08-22T02:17:45.935Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 782.781139ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2726 | 2024-08-22T02:17:45.943Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 1.096978691s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2727 | 2024-08-22T02:17:46.064Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 1.261196138s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2728 | 2024-08-22T02:17:46.076Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 930.597437ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2729 | 2024-08-22T02:17:46.119Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 1.467968443s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2730 | 2024-08-22T02:17:46.199Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 1.486658554s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2731 | 2024-08-22T02:17:46.208Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 1.444632174s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2732 | 2024-08-22T02:17:46.239Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 643.698851ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2733 | 2024-08-22T02:17:46.405Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 1.217923249s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2734 | 2024-08-22T02:17:46.483Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 905.114423ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2735 | 2024-08-22T02:17:46.532Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 1.144254019s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2736 | 2024-08-22T02:17:46.559Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 511.682218ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2737 | 2024-08-22T02:17:46.595Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 1.280865756s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2738 | 2024-08-22T02:17:46.626Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 1.302075138s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2739 | 2024-08-22T02:17:46.731Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 1.383467093s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2740 | 2024-08-22T02:17:46.899Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 1.036636791s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2741 | 2024-08-22T02:17:46.973Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 649.954274ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2742 | 2024-08-22T02:17:46.990Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 1.325813866s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2743 | 2024-08-22T02:17:47.004Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 1.09198639s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2744 | 2024-08-22T02:17:47.020Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 936.46395ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2745 | 2024-08-22T02:17:47.056Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 925.878621ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2746 | 2024-08-22T02:17:47.069Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 663.324896ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2747 | 2024-08-22T02:17:47.083Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 595.772566ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2748 | 2024-08-22T02:17:47.144Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 868.748925ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2749 | 2024-08-22T02:17:47.339Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 1.335786599s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2750 | 2024-08-22T02:17:47.400Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 635.101603ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2751 | 2024-08-22T02:17:47.604Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 626.959441ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2752 | 2024-08-22T02:17:47.637Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 1.126583869s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2753 | 2024-08-22T02:17:47.654Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 1.095387239s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2754 | 2024-08-22T02:17:47.665Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 530.401358ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2755 | 2024-08-22T02:17:47.699Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 906.500405ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2756 | 2024-08-22T02:17:47.713Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 989.26728ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2757 | 2024-08-22T02:17:47.728Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 1.076399562s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2758 | 2024-08-22T02:17:47.747Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 680.077884ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2759 | 2024-08-22T02:17:47.892Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 1.212917684s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2760 | 2024-08-22T02:17:47.943Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 633.502353ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2761 | 2024-08-22T02:17:47.957Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 1.009396401s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2762 | 2024-08-22T02:17:47.969Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 900.936872ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2763 | 2024-08-22T02:17:47.994Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 911.923291ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2764 | 2024-08-22T02:17:48.024Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 854.449983ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2765 | 2024-08-22T02:17:48.048Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 1.306612445s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2766 | 2024-08-22T02:17:48.111Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 1.36467335s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2767 | 2024-08-22T02:17:48.127Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 729.335324ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2768 | 2024-08-22T02:17:48.214Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 1.014361201s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2769 | 2024-08-22T02:17:48.245Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 1.160101113s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2770 | 2024-08-22T02:17:48.331Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 1.097459632s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2771 | 2024-08-22T02:17:48.447Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 770.421447ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2772 | 2024-08-22T02:17:48.589Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 1.34443126s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2773 | 2024-08-22T02:17:48.622Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 803.208721ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2774 | 2024-08-22T02:17:48.689Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 526.262226ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2775 | 2024-08-22T02:17:48.718Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 861.914394ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2776 | 2024-08-22T02:17:48.764Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 1.02341559s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2777 | 2024-08-22T02:17:48.777Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 615.822059ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2778 | 2024-08-22T02:17:48.820Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 1.019184628s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2779 | 2024-08-22T02:17:48.870Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 1.252348474s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2780 | 2024-08-22T02:17:48.881Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 1.08400127s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2781 | 2024-08-22T02:17:48.891Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 546.130708ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2782 | 2024-08-22T02:17:48.919Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 1.453729521s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2783 | 2024-08-22T02:17:48.977Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 986.276799ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2784 | 2024-08-22T02:17:49.118Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 1.151268433s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2785 | 2024-08-22T02:17:49.227Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 1.381242651s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2786 | 2024-08-22T02:17:49.239Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 1.288084552s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2787 | 2024-08-22T02:17:49.250Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 771.262377ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2788 | 2024-08-22T02:17:49.366Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 670.965466ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2789 | 2024-08-22T02:17:49.404Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 1.069884085s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2790 | 2024-08-22T02:17:49.417Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 872.779044ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2791 | 2024-08-22T02:17:49.436Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 1.28535344s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2792 | 2024-08-22T02:17:49.447Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 660.932699ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2793 | 2024-08-22T02:17:49.458Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 1.474379619s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2794 | 2024-08-22T02:17:49.488Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 913.51986ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2795 | 2024-08-22T02:17:49.592Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 781.485405ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2796 | 2024-08-22T02:17:49.802Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 533.031926ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2797 | 2024-08-22T02:17:49.851Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 950.150883ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2798 | 2024-08-22T02:17:49.946Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 1.092973476s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2799 | 2024-08-22T02:17:49.975Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 1.045571842s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2800 | 2024-08-22T02:17:49.987Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 1.018058603s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2801 | 2024-08-22T02:17:50.032Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 654.359921ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2802 | 2024-08-22T02:17:50.047Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 1.434591245s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2803 | 2024-08-22T02:17:50.122Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 545.092955ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2804 | 2024-08-22T02:17:50.141Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 1.129392144s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2805 | 2024-08-22T02:17:50.285Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 925.65334ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2806 | 2024-08-22T02:17:50.305Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 994.201807ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2807 | 2024-08-22T02:17:50.350Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 697.266175ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2808 | 2024-08-22T02:17:50.388Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 817.672077ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2809 | 2024-08-22T02:17:50.402Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 1.409938213s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2810 | 2024-08-22T02:17:50.415Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 972.297569ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2811 | 2024-08-22T02:17:50.488Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 610.03959ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2812 | 2024-08-22T02:17:50.540Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 1.282719656s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2813 | 2024-08-22T02:17:50.623Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 702.275399ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2814 | 2024-08-22T02:17:50.685Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 1.322839873s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2815 | 2024-08-22T02:17:50.699Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 1.376293802s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2816 | 2024-08-22T02:17:50.738Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 733.683823ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2817 | 2024-08-22T02:17:50.816Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 935.909057ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2818 | 2024-08-22T02:17:50.946Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 1.22499581s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2819 | 2024-08-22T02:17:51.021Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 1.366467436s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2820 | 2024-08-22T02:17:51.033Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 972.003935ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2821 | 2024-08-22T02:17:51.050Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 1.343474741s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2822 | 2024-08-22T02:17:51.060Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 1.12341532s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2823 | 2024-08-22T02:17:51.113Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 1.35122527s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2824 | 2024-08-22T02:17:51.219Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 976.140682ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2825 | 2024-08-22T02:17:51.234Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2") failed: Property not found. retry in 1.116717715s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2826 | 2024-08-22T02:17:51.284Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 683.663866ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2827 | 2024-08-22T02:17:51.314Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 742.730669ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2828 | 2024-08-22T02:17:51.341Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 697.282179ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2829 | 2024-08-22T02:17:51.400Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 1.149843867s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2830 | 2024-08-22T02:17:51.487Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 1.390876162s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2831 | 2024-08-22T02:17:51.504Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864") failed: Property not found. retry in 576.152045ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2832 | 2024-08-22T02:17:51.618Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:64126
|
| 2833 | 2024-08-22T02:17:51.619Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 527
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:64126
req_id = 84a292e7-1ef0-48f1-a6fe-62714a13d625
response_code = 200
uri = /baseboard
|
| 2834 | 2024-08-22T02:17:51.770Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 964.616733ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2835 | 2024-08-22T02:17:51.833Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 604.195835ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2836 | 2024-08-22T02:17:51.847Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 1.060325612s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2837 | 2024-08-22T02:17:51.980Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 1.15133976s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2838 | 2024-08-22T02:17:52.015Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 990.910657ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2839 | 2024-08-22T02:17:52.023Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 568.874745ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2840 | 2024-08-22T02:17:52.048Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 955.363157ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2841 | 2024-08-22T02:17:52.067Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 716.23751ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2842 | 2024-08-22T02:17:52.086Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 1.204146892s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2843 | 2024-08-22T02:17:52.185Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 901.140486ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2844 | 2024-08-22T02:17:52.193Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df") failed: Property not found. retry in 828.455119ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2845 | 2024-08-22T02:17:52.205Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 1.194417775s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2846 | 2024-08-22T02:17:52.399Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906") failed: Property not found. retry in 674.544122ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2847 | 2024-08-22T02:17:52.408Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46") failed: Property not found. retry in 1.495457495s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2848 | 2024-08-22T02:17:52.448Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 1.108234098s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2849 | 2024-08-22T02:17:52.474Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 1.048132309s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2850 | 2024-08-22T02:17:52.568Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 931.695612ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2851 | 2024-08-22T02:17:52.602Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4") failed: Property not found. retry in 1.290582048s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2852 | 2024-08-22T02:17:52.746Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 735.466934ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2853 | 2024-08-22T02:17:52.792Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c") failed: Property not found. retry in 855.45622ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2854 | 2024-08-22T02:17:52.890Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49") failed: Property not found. retry in 773.979502ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2855 | 2024-08-22T02:17:52.920Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959") failed: Property not found. retry in 1.080095311s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2856 | 2024-08-22T02:17:53.014Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 544.668133ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2857 | 2024-08-22T02:17:53.023Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 614.514004ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2858 | 2024-08-22T02:17:53.098Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576") failed: Property not found. retry in 1.356751088s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2859 | 2024-08-22T02:17:53.145Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33") failed: Property not found. retry in 1.43307313s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2860 | 2024-08-22T02:17:53.308Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3") failed: Property not found. retry in 796.021331ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2861 | 2024-08-22T02:17:53.414Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf") failed: Property not found. retry in 1.101290036s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2862 | 2024-08-22T02:17:53.495Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d") failed: Property not found. retry in 1.106427106s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2863 | 2024-08-22T02:17:53.511Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f") failed: Property not found. retry in 1.275976221s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2864 | 2024-08-22T02:17:53.534Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50") failed: Property not found. retry in 1.493128797s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2865 | 2024-08-22T02:17:53.568Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7") failed: Property not found. retry in 1.264084516s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2866 | 2024-08-22T02:17:53.578Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7") failed: Property not found. retry in 703.558201ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2867 | 2024-08-22T02:17:53.657Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f") failed: Property not found. retry in 882.396211ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2868 | 2024-08-22T02:17:54.219Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 18.40503651s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = oximeter/producer/src/lib.rs:424
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 2869 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_nexus_7524dcb2-fbae-4cac-83ef-d3e4411c247f
|
| 2870 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_nexus_95247a59-463e-4d62-a4ab-f35b1fbfb959
|
| 2871 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_crucible_2b4e4c46-69fb-47e0-bfb6-01d380971bb2
|
| 2872 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_crucible_7e931c9a-fc76-4509-b7db-fa4aa9e03576
|
| 2873 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_crucible_bf20743c-2c90-4d79-8cba-0a0d296d31a7
|
| 2874 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_external_dns_e3cfd6ed-225d-4c9e-a437-60c94e80df50
|
| 2875 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_nexus_249b0525-e315-49f0-bea0-213c52bdfd49
|
| 2876 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_clickhouse_e604cd42-8d92-4bb5-b07a-6931fdbf2ec7
|
| 2877 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_external_dns_7d0db839-e1b4-4b99-8b58-7f70a3941e8d
|
| 2878 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_crucible_pantry_a62caba8-947d-463e-8485-54a699632f6c
|
| 2879 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_crucible_d1073410-4570-439e-a9f7-a33b9380747f
|
| 2880 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_crucible_dce94a66-3e8e-4ef2-a278-14fcbaefb3df
|
| 2881 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_crucible_8e7742e8-523b-430d-a69a-9f9f4d4a7adf
|
| 2882 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_oximeter_83b9c234-fcfa-4fe9-b4b7-c61db9754b46
|
| 2883 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_crucible_pantry_383c9379-db34-4f35-bbb7-0ff693eb5864
|
| 2884 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_crucible_pantry_60d227cb-8f8d-4ba6-8fed-61d664284906
|
| 2885 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_crucible_a515946e-cb4e-4ebb-b0f1-eea57fa935f3
|
| 2886 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_crucible_fc052a6b-ed5e-4aa6-b998-97446b9d35e4
|
| 2887 | 2024-08-22T02:17:55.063Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3181
zone = oxz_crucible_558fb2d9-3acc-4a64-bff5-f85a077f1e33
|
| 2888 | 2024-08-22T02:17:55.064Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 2889 | 2024-08-22T02:17:55.067Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 2890 | 2024-08-22T02:17:55.087Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 37058106
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47684
req_id = 118c07f1-8b16-423a-ac03-f3be1b7c5d86
response_code = 204
uri = /omicron-zones
|
| 2891 | 2024-08-22T02:17:55.096Z | INFO | SledAgent (RSS): Finished setting up services
file = sled-agent/src/rack_setup/service.rs:1251
|
| 2892 | 2024-08-22T02:17:55.096Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/.rss-plan-completed.marker.tmp
file = common/src/ledger.rs:199
|
| 2893 | 2024-08-22T02:17:55.096Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/.rss-plan-completed.marker.tmp
file = common/src/ledger.rs:199
|
| 2894 | 2024-08-22T02:17:55.109Z | INFO | SledAgent (RSS): Handing off control to Nexus
file = sled-agent/src/rack_setup/service.rs:688
|
| 2895 | 2024-08-22T02:17:55.112Z | INFO | SledAgent (RSS): Nexus address: [fd00:1122:3344:101::a]:12221
file = sled-agent/src/rack_setup/service.rs:701
|
| 2896 | 2024-08-22T02:17:55.147Z | INFO | SledAgent (RSS): rack_network_config: RackNetworkConfigV2 {
bfd: [],
bgp: [],
infra_ip_first: 10.151.2.174,
infra_ip_last: 10.151.2.174,
ports: [
PortConfigV2 {
addresses: [
UplinkAddressConfig {
address: V4(
Ipv4Net {
addr: 10.151.2.174,
width: 24,
},
),
vlan_id: None,
},
],
autoneg: false,
bgp_peers: [],
port: "qsfp0",
routes: [
RouteConfig {
destination: V4(
Ipv4Net {
addr: 0.0.0.0,
width: 0,
},
),
nexthop: 10.151.2.100,
vlan_id: None,
},
],
switch: Switch0,
uplink_port_fec: None,
uplink_port_speed: Speed40G,
},
],
rack_subnet: Ipv6Net {
addr: fd00:1122:3344:100::,
width: 56,
},
}
file = sled-agent/src/rack_setup/service.rs:832
|
| 2897 | 2024-08-22T02:17:55.149Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2898 | 2024-08-22T02:17:55.290Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2899 | 2024-08-22T02:17:55.383Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2900 | 2024-08-22T02:17:55.561Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2901 | 2024-08-22T02:17:55.735Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2902 | 2024-08-22T02:17:55.998Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2903 | 2024-08-22T02:17:56.331Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2904 | 2024-08-22T02:17:56.691Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2905 | 2024-08-22T02:17:57.147Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2906 | 2024-08-22T02:17:57.785Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2907 | 2024-08-22T02:17:58.406Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2908 | 2024-08-22T02:17:59.300Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2909 | 2024-08-22T02:17:59.745Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2910 | 2024-08-22T02:18:00.559Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2911 | 2024-08-22T02:18:01.367Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2912 | 2024-08-22T02:18:02.218Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2913 | 2024-08-22T02:18:03.765Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2914 | 2024-08-22T02:18:06.001Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/807d2f15-4cac-40e5-8f79-4faf50cf2768/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:894
|
| 2915 | 2024-08-22T02:18:07.883Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "19e6584f-0189-4a41-8aee-4e4824101920", "content-length": "173", "date": "Thu, 22 Aug 2024 02:18:07 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"37fb4e95-8c94-4aa8-946d-de6b4770caf9\"", request_id: "19e6584f-0189-4a41-8aee-4e4824101920" }
file = sled-agent/src/rack_setup/service.rs:894
|
| 2916 | 2024-08-22T02:18:11.882Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "89400715-eb73-4cc8-bffb-11ab6351921b", "content-length": "173", "date": "Thu, 22 Aug 2024 02:18:11 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"37fb4e95-8c94-4aa8-946d-de6b4770caf9\"", request_id: "89400715-eb73-4cc8-bffb-11ab6351921b" }
file = sled-agent/src/rack_setup/service.rs:894
|
| 2917 | 2024-08-22T02:18:12.596Z | INFO | SledAgent (NexusNotifierTask): Successfully put SledAgentInfo to nexus
file = sled-agent/src/nexus.rs:544
new_generation = 1
old_generation = 1
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 2918 | 2024-08-22T02:18:12.684Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:43250
|
| 2919 | 2024-08-22T02:18:12.687Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2920 | 2024-08-22T02:18:12.776Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 91925
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:43250
req_id = 10fa790e-6878-4603-b05b-9ee0a131a18f
response_code = 200
uri = /inventory
|
| 2921 | 2024-08-22T02:18:12.779Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2922 | 2024-08-22T02:18:12.779Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2923 | 2024-08-22T02:18:12.781Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2924 | 2024-08-22T02:18:12.782Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 3725
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:43250
req_id = a19972ff-1758-4a0f-8150-c3accba27822
response_code = 200
uri = /omicron-zones
|
| 2925 | 2024-08-22T02:18:14.558Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:56423
|
| 2926 | 2024-08-22T02:18:14.564Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:671
rules = [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2927 | 2024-08-22T02:18:14.564Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2928 | 2024-08-22T02:18:14.564Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2929 | 2024-08-22T02:18:14.564Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2930 | 2024-08-22T02:18:14.564Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte0"
rules = []
|
| 2931 | 2024-08-22T02:18:14.564Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2932 | 2024-08-22T02:18:14.564Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2933 | 2024-08-22T02:18:14.564Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 6569
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:56423
req_id = bbcacf7e-f4a4-4301-a9e0-17082173e46c
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2934 | 2024-08-22T02:18:14.566Z | INFO | SledAgent (RSS): Handoff to Nexus is complete
file = sled-agent/src/rack_setup/service.rs:904
|
| 2935 | 2024-08-22T02:18:14.874Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:51027
|
| 2936 | 2024-08-22T02:18:14.874Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2937 | 2024-08-22T02:18:14.969Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 95031
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:51027
req_id = 7951a617-7c86-4301-ad65-add58b30f8f9
response_code = 200
uri = /inventory
|
| 2938 | 2024-08-22T02:18:14.970Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 2939 | 2024-08-22T02:18:14.970Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2940 | 2024-08-22T02:18:14.970Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2941 | 2024-08-22T02:18:14.971Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1208
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:51027
req_id = 9d2300a0-b343-4700-abfc-bad8a517e54a
response_code = 200
uri = /omicron-zones
|
| 2942 | 2024-08-22T02:18:16.205Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:44303
|
| 2943 | 2024-08-22T02:18:16.206Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:671
rules = [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2944 | 2024-08-22T02:18:16.206Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2945 | 2024-08-22T02:18:16.206Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2946 | 2024-08-22T02:18:16.206Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2947 | 2024-08-22T02:18:16.206Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte0"
rules = []
|
| 2948 | 2024-08-22T02:18:16.206Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2949 | 2024-08-22T02:18:16.206Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2950 | 2024-08-22T02:18:16.206Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 318
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:44303
req_id = 13240c2a-ba47-47e4-ac29-322fab2cb8b3
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2951 | 2024-08-22T02:18:16.269Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:43000
|
| 2952 | 2024-08-22T02:18:16.269Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:671
rules = [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2953 | 2024-08-22T02:18:16.269Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2954 | 2024-08-22T02:18:16.269Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2955 | 2024-08-22T02:18:16.269Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2956 | 2024-08-22T02:18:16.269Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte0"
rules = []
|
| 2957 | 2024-08-22T02:18:16.269Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2958 | 2024-08-22T02:18:16.269Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2959 | 2024-08-22T02:18:16.269Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 233
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:43000
req_id = a4684c2c-1a3d-4d1d-9470-4b1c72ef3951
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2960 | 2024-08-22T02:18:16.386Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:44596
|
| 2961 | 2024-08-22T02:18:16.386Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:671
rules = [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2962 | 2024-08-22T02:18:16.386Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2963 | 2024-08-22T02:18:16.386Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2964 | 2024-08-22T02:18:16.386Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2965 | 2024-08-22T02:18:16.386Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte0"
rules = []
|
| 2966 | 2024-08-22T02:18:16.386Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2967 | 2024-08-22T02:18:16.386Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2968 | 2024-08-22T02:18:16.386Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 248
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:44596
req_id = 0df40a61-7d01-47db-8300-09cbdd5aac2b
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2969 | 2024-08-22T02:18:20.680Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:65268
|
| 2970 | 2024-08-22T02:18:20.681Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 591
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:65268
req_id = c892a966-5ef0-4aa1-bf6b-81edbf3633fb
response_code = 200
uri = /vpc-routes
|
| 2971 | 2024-08-22T02:18:20.741Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1743
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:65268
req_id = a83bb3c2-00cd-4915-8439-ac7d9580dace
response_code = 204
uri = /vpc-routes
|
| 2972 | 2024-08-22T02:18:21.657Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:58140
|
| 2973 | 2024-08-22T02:18:21.657Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 71
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:58140
req_id = a6044695-bc27-4937-bbc3-d73d1104369f
response_code = 200
uri = /baseboard
|
| 2974 | 2024-08-22T02:18:36.886Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:42738
|
| 2975 | 2024-08-22T02:18:36.887Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1156
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:42738
req_id = d46855ce-eee8-4bdc-9d94-c15414b452d7
response_code = 200
uri = /vpc-routes
|
| 2976 | 2024-08-22T02:18:36.912Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 154
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:42738
req_id = e780b714-6f70-4f94-9e76-fa70379104e2
response_code = 204
uri = /vpc-routes
|
| 2977 | 2024-08-22T02:18:37.030Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:41142
|
| 2978 | 2024-08-22T02:18:37.030Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 60
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:41142
req_id = b5a45788-6f45-4a15-99a2-18e2cb799132
response_code = 200
uri = /vpc-routes
|
| 2979 | 2024-08-22T02:18:37.043Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1066
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:41142
req_id = d7c7efa5-8065-42f8-a93f-7b52201bdd59
response_code = 204
uri = /vpc-routes
|
| 2980 | 2024-08-22T02:18:37.053Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:38713
|
| 2981 | 2024-08-22T02:18:37.054Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 969
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:38713
req_id = 14f83794-d376-4ca6-bab3-c57cdc15f400
response_code = 200
uri = /v2p
|
| 2982 | 2024-08-22T02:18:37.065Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:44422
|
| 2983 | 2024-08-22T02:18:37.065Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 33
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:44422
req_id = a0792a59-a262-4d69-b9c9-653b34327e4f
response_code = 200
uri = /vpc-routes
|
| 2984 | 2024-08-22T02:18:37.074Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 72
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:44422
req_id = 50cd4c7c-ac4a-4765-9585-0a57387ba31a
response_code = 204
uri = /vpc-routes
|
| 2985 | 2024-08-22T02:18:37.085Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:60379
|
| 2986 | 2024-08-22T02:18:37.086Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 726
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:60379
req_id = b77130df-5d5a-4cc6-8ac1-8054967891c2
response_code = 200
uri = /v2p
|
| 2987 | 2024-08-22T02:18:37.142Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:62689
|
| 2988 | 2024-08-22T02:18:37.142Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 111
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:62689
req_id = 6e19f971-4526-4dd4-9181-fa577ac7153e
response_code = 200
uri = /v2p
|
| 2989 | 2024-08-22T02:18:37.172Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:64252
|
| 2990 | 2024-08-22T02:18:37.173Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 2991 | 2024-08-22T02:18:37.244Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 2992 | 2024-08-22T02:18:37.271Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 98349
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:64252
req_id = b3aaae69-d2a8-46ce-870c-c27fe705b584
response_code = 204
uri = /switch-ports
|
| 2993 | 2024-08-22T02:18:37.271Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:32808
|
| 2994 | 2024-08-22T02:18:37.271Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:35450
|
| 2995 | 2024-08-22T02:18:37.272Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 2996 | 2024-08-22T02:18:37.275Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 4069
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:64252
req_id = 6f2214d1-8b2d-47a7-b1ac-832308ac6d53
response_code = 200
uri = /network-bootstore-config
|
| 2997 | 2024-08-22T02:18:37.300Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 2998 | 2024-08-22T02:18:37.310Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 2, current_generation: 1
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
| 2999 | 2024-08-22T02:18:37.311Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 3000 | 2024-08-22T02:18:37.312Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 3001 | 2024-08-22T02:18:37.313Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 2
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-sock-i86pc
|
| 3002 | 2024-08-22T02:18:37.313Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 3597
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:64252
req_id = 1071873a-2834-44b0-afd6-f18631333edf
response_code = 204
uri = /network-bootstore-config
|
| 3003 | 2024-08-22T02:18:37.336Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 64626
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:32808
req_id = eb49a83d-59a0-4660-9c72-6299e8f80faa
response_code = 204
uri = /switch-ports
|
| 3004 | 2024-08-22T02:18:37.336Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3005 | 2024-08-22T02:18:37.341Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1168
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:32808
req_id = 578a0feb-60ac-4ab0-85fe-6ddc2d1dedaf
response_code = 200
uri = /network-bootstore-config
|
| 3006 | 2024-08-22T02:18:37.364Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3007 | 2024-08-22T02:18:37.387Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 116227
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:35450
req_id = bd646a7a-0389-4fb3-864b-263cc365aa08
response_code = 204
uri = /switch-ports
|
| 3008 | 2024-08-22T02:18:37.388Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 136
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:35450
req_id = 202252ce-31bc-4d6d-b635-e01839b0fb09
response_code = 200
uri = /network-bootstore-config
|
| 3009 | 2024-08-22T02:18:42.764Z | INFO | SledAgent (dropshot): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:42716
remote_addr = [fd00:1122:3344:101::d]:54144
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3010 | 2024-08-22T02:18:42.795Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 31142
local_addr = [fd00:1122:3344:101::1]:42716
method = GET
remote_addr = [fd00:1122:3344:101::d]:54144
req_id = 7049dcee-7cf2-401a-ad81-e4daf78a1345
response_code = 200
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
uri = /37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3011 | 2024-08-22T02:18:51.703Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:55945
|
| 3012 | 2024-08-22T02:18:51.703Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 51
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:55945
req_id = 6a2b3e19-36ba-44c8-95f9-5f6759cc629c
response_code = 200
uri = /baseboard
|
| 3013 | 2024-08-22T02:19:06.914Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:61260
|
| 3014 | 2024-08-22T02:19:06.914Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 243
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:61260
req_id = f83d8c2f-741f-44fa-879d-fc45db4a44f1
response_code = 200
uri = /v2p
|
| 3015 | 2024-08-22T02:19:07.106Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:52819
|
| 3016 | 2024-08-22T02:19:07.106Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 174
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:52819
req_id = 54f75972-3f00-43c1-a4b1-2b78fa46c03f
response_code = 200
uri = /v2p
|
| 3017 | 2024-08-22T02:19:07.145Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:34891
|
| 3018 | 2024-08-22T02:19:07.145Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 78
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:34891
req_id = 86e7c487-c3cd-40d3-8cba-9e7ff3b547a8
response_code = 200
uri = /vpc-routes
|
| 3019 | 2024-08-22T02:19:07.156Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 132
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:34891
req_id = bca25fd4-af77-4410-b627-50a109239873
response_code = 204
uri = /vpc-routes
|
| 3020 | 2024-08-22T02:19:07.188Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:48445
|
| 3021 | 2024-08-22T02:19:07.189Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 126
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:48445
req_id = bc3b9288-ed3c-4637-af57-e2114f0ae981
response_code = 200
uri = /v2p
|
| 3022 | 2024-08-22T02:19:07.190Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:61668
|
| 3023 | 2024-08-22T02:19:07.190Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 33
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:61668
req_id = 8f49eeea-f042-48ba-ba16-806584ede62e
response_code = 200
uri = /vpc-routes
|
| 3024 | 2024-08-22T02:19:07.198Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 76
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:61668
req_id = c19adc8d-1b5c-47a3-a0cb-e94265dda214
response_code = 204
uri = /vpc-routes
|
| 3025 | 2024-08-22T02:19:07.310Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:38811
|
| 3026 | 2024-08-22T02:19:07.311Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 934
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:38811
req_id = 05d24d7b-39b7-4884-b0fe-295bac4f6e40
response_code = 200
uri = /vpc-routes
|
| 3027 | 2024-08-22T02:19:07.320Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 99
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:38811
req_id = 91282cb6-ee0f-4fd4-ace5-767f1eff9867
response_code = 204
uri = /vpc-routes
|
| 3028 | 2024-08-22T02:19:07.337Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:42261
|
| 3029 | 2024-08-22T02:19:07.337Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3030 | 2024-08-22T02:19:07.362Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3031 | 2024-08-22T02:19:07.384Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 47109
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:42261
req_id = a6bd5ac0-402b-4201-b163-10958a3c7edf
response_code = 204
uri = /switch-ports
|
| 3032 | 2024-08-22T02:19:07.384Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 105
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:42261
req_id = a14dcd61-f1d0-4e86-a7c3-47de910353fc
response_code = 200
uri = /network-bootstore-config
|
| 3033 | 2024-08-22T02:19:07.417Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:63593
|
| 3034 | 2024-08-22T02:19:07.417Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3035 | 2024-08-22T02:19:07.441Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3036 | 2024-08-22T02:19:07.462Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 44473
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:63593
req_id = 55c07318-5a77-4930-ad1e-5bdfa81d9177
response_code = 204
uri = /switch-ports
|
| 3037 | 2024-08-22T02:19:07.462Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 99
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:63593
req_id = a641c6b1-e691-4c83-8f04-7909ef0bc38b
response_code = 200
uri = /network-bootstore-config
|
| 3038 | 2024-08-22T02:19:07.732Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:61799
|
| 3039 | 2024-08-22T02:19:07.732Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3040 | 2024-08-22T02:19:07.757Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3041 | 2024-08-22T02:19:07.780Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 48136
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:61799
req_id = 769b5631-83d2-4770-ae46-36d701095dd2
response_code = 204
uri = /switch-ports
|
| 3042 | 2024-08-22T02:19:07.781Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 101
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:61799
req_id = 2b6d20dc-3fc0-42f4-886c-9c6a4cc04007
response_code = 200
uri = /network-bootstore-config
|
| 3043 | 2024-08-22T02:19:12.770Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 5940
local_addr = [fd00:1122:3344:101::1]:42716
method = GET
remote_addr = [fd00:1122:3344:101::d]:54144
req_id = 21c5484d-f01b-4078-8bbe-8646a65d54c7
response_code = 200
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
uri = /37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3044 | 2024-08-22T02:19:21.738Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:58034
|
| 3045 | 2024-08-22T02:19:21.738Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 41
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:58034
req_id = 321c2ba1-d42d-485e-8183-87fc483e19bc
response_code = 200
uri = /baseboard
|
| 3046 | 2024-08-22T02:19:36.918Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:55312
|
| 3047 | 2024-08-22T02:19:36.919Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 138
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:55312
req_id = 1e95679b-762c-4898-9e38-19c608e645bb
response_code = 200
uri = /vpc-routes
|
| 3048 | 2024-08-22T02:19:36.928Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 134
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:55312
req_id = fed588ef-0bf2-4890-b950-b1644795ca20
response_code = 204
uri = /vpc-routes
|
| 3049 | 2024-08-22T02:19:37.004Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:62924
|
| 3050 | 2024-08-22T02:19:37.005Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 933
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:62924
req_id = 0cc7c3a2-968b-472b-be65-648e4e549fb4
response_code = 200
uri = /v2p
|
| 3051 | 2024-08-22T02:19:37.022Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:32888
|
| 3052 | 2024-08-22T02:19:37.022Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 86
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:32888
req_id = efc495c8-b6b9-4793-b776-296aa51a1f08
response_code = 200
uri = /v2p
|
| 3053 | 2024-08-22T02:19:37.032Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:39558
|
| 3054 | 2024-08-22T02:19:37.032Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 42
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:39558
req_id = cebe6bec-004a-4f19-952b-c7dbe6a6c48a
response_code = 200
uri = /vpc-routes
|
| 3055 | 2024-08-22T02:19:37.041Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 93
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:39558
req_id = 333cf8af-6d4d-4d30-a62e-7cf13b5d2410
response_code = 204
uri = /vpc-routes
|
| 3056 | 2024-08-22T02:19:37.115Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:52272
|
| 3057 | 2024-08-22T02:19:37.115Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3058 | 2024-08-22T02:19:37.142Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3059 | 2024-08-22T02:19:37.167Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 51128
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:52272
req_id = 35f0c845-7dc6-4bcd-afd8-b3c21a2d662d
response_code = 204
uri = /switch-ports
|
| 3060 | 2024-08-22T02:19:37.167Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:51792
|
| 3061 | 2024-08-22T02:19:37.168Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3062 | 2024-08-22T02:19:37.168Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 927
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:52272
req_id = a7d086ce-5867-482f-91a2-4221b0200ac2
response_code = 200
uri = /network-bootstore-config
|
| 3063 | 2024-08-22T02:19:37.174Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:52699
|
| 3064 | 2024-08-22T02:19:37.174Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 94
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:52699
req_id = 2a732dc4-1d94-4eea-ae01-f9c05af7309f
response_code = 200
uri = /vpc-routes
|
| 3065 | 2024-08-22T02:19:37.185Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 563
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:52699
req_id = 70c941c5-dbb6-4b77-aa30-f4c1a6474a3b
response_code = 204
uri = /vpc-routes
|
| 3066 | 2024-08-22T02:19:37.193Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3067 | 2024-08-22T02:19:37.213Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 46406
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:51792
req_id = a6945615-a222-4230-937f-d84de7d4d488
response_code = 204
uri = /switch-ports
|
| 3068 | 2024-08-22T02:19:37.214Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 645
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:51792
req_id = 881434a6-bead-4aa9-b4b6-1269f348560e
response_code = 200
uri = /network-bootstore-config
|
| 3069 | 2024-08-22T02:19:37.254Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:53287
|
| 3070 | 2024-08-22T02:19:37.255Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 155
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:53287
req_id = 4df6ee82-6103-4251-b268-c8982f4c28f6
response_code = 200
uri = /v2p
|
| 3071 | 2024-08-22T02:19:37.414Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:47199
|
| 3072 | 2024-08-22T02:19:37.414Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3073 | 2024-08-22T02:19:37.440Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3074 | 2024-08-22T02:19:37.461Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 46496
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:47199
req_id = 60740d09-3276-4c45-9852-5c7833ee6afe
response_code = 204
uri = /switch-ports
|
| 3075 | 2024-08-22T02:19:37.461Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 103
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:47199
req_id = 7464798d-5f23-4ac0-8388-c460d27d3b24
response_code = 200
uri = /network-bootstore-config
|
| 3076 | 2024-08-22T02:19:42.774Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 10073
local_addr = [fd00:1122:3344:101::1]:42716
method = GET
remote_addr = [fd00:1122:3344:101::d]:54144
req_id = 3a485889-413a-4c3e-ae03-a42894753d5b
response_code = 200
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
uri = /37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3077 | 2024-08-22T02:19:51.773Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56551
|
| 3078 | 2024-08-22T02:19:51.773Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 48
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56551
req_id = 056cc82c-138e-4ce4-953a-028022a09525
response_code = 200
uri = /baseboard
|
| 3079 | 2024-08-22T02:20:07.047Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:37762
|
| 3080 | 2024-08-22T02:20:07.047Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 106
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:37762
req_id = 4a7445bf-3d66-47ad-aea8-9c8a560aa640
response_code = 200
uri = /vpc-routes
|
| 3081 | 2024-08-22T02:20:07.060Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1422
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:37762
req_id = 54466ffc-b69c-4327-b5a2-45c8a13e9b7c
response_code = 204
uri = /vpc-routes
|
| 3082 | 2024-08-22T02:20:07.096Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:64541
|
| 3083 | 2024-08-22T02:20:07.096Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 136
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:64541
req_id = 46711cb2-d567-40f6-80cd-b22496ddad4d
response_code = 200
uri = /v2p
|
| 3084 | 2024-08-22T02:20:07.187Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:56861
|
| 3085 | 2024-08-22T02:20:07.188Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1301
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:56861
req_id = 97f51b7e-5081-43be-ab4b-65a7b67fa0d7
response_code = 200
uri = /v2p
|
| 3086 | 2024-08-22T02:20:07.271Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:65250
|
| 3087 | 2024-08-22T02:20:07.271Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 37
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:65250
req_id = e6851c64-578f-481d-9631-e07c29aae5c4
response_code = 200
uri = /vpc-routes
|
| 3088 | 2024-08-22T02:20:07.279Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:45671
|
| 3089 | 2024-08-22T02:20:07.279Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 91
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:45671
req_id = 17bae8a0-72d8-43cb-b90c-2123809d9084
response_code = 200
uri = /v2p
|
| 3090 | 2024-08-22T02:20:07.281Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 71
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:65250
req_id = ccbdf315-3f92-4570-a694-7797cb4b0fb7
response_code = 204
uri = /vpc-routes
|
| 3091 | 2024-08-22T02:20:07.335Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:59103
|
| 3092 | 2024-08-22T02:20:07.336Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 291
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:59103
req_id = 7bb053be-cca4-4f26-9136-39080100a375
response_code = 200
uri = /vpc-routes
|
| 3093 | 2024-08-22T02:20:07.336Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:64078
|
| 3094 | 2024-08-22T02:20:07.336Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3095 | 2024-08-22T02:20:07.361Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3096 | 2024-08-22T02:20:07.384Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 47620
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:64078
req_id = d1f96ea1-035b-47ce-a9be-d58c4e364d58
response_code = 204
uri = /switch-ports
|
| 3097 | 2024-08-22T02:20:07.384Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 92
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:59103
req_id = bc8bfd6f-6124-4394-9473-de2100a5a386
response_code = 204
uri = /vpc-routes
|
| 3098 | 2024-08-22T02:20:07.384Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 80
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:64078
req_id = 24b75d66-4275-4235-bb5b-91f3722783bd
response_code = 200
uri = /network-bootstore-config
|
| 3099 | 2024-08-22T02:20:07.414Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:36197
|
| 3100 | 2024-08-22T02:20:07.414Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3101 | 2024-08-22T02:20:07.438Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3102 | 2024-08-22T02:20:07.460Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 45681
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:36197
req_id = 6a4a8b77-3c6c-49b3-b915-17ab4fa0ee8f
response_code = 204
uri = /switch-ports
|
| 3103 | 2024-08-22T02:20:07.460Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:33304
|
| 3104 | 2024-08-22T02:20:07.460Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3105 | 2024-08-22T02:20:07.486Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3106 | 2024-08-22T02:20:07.510Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 50386
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:33304
req_id = de38bda3-9623-49e6-890d-d55b543862c1
response_code = 204
uri = /switch-ports
|
| 3107 | 2024-08-22T02:20:07.511Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1186
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:36197
req_id = 24cc1ed0-69dc-4629-83dc-a2463958b8f6
response_code = 200
uri = /network-bootstore-config
|
| 3108 | 2024-08-22T02:20:07.512Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 134
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:33304
req_id = ce8a5fa4-3bfc-4268-8272-784c8b4703f5
response_code = 200
uri = /network-bootstore-config
|
| 3109 | 2024-08-22T02:20:12.771Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 5652
local_addr = [fd00:1122:3344:101::1]:42716
method = GET
remote_addr = [fd00:1122:3344:101::d]:54144
req_id = 08142436-911d-4653-88e6-80045668e907
response_code = 200
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
uri = /37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3110 | 2024-08-22T02:20:21.808Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:39777
|
| 3111 | 2024-08-22T02:20:21.808Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 44
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:39777
req_id = 10d296e7-f34a-4b52-bf8e-7aedcd785a72
response_code = 200
uri = /baseboard
|
| 3112 | 2024-08-22T02:20:36.916Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:35316
|
| 3113 | 2024-08-22T02:20:36.916Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 293
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:35316
req_id = 7ab448bb-9205-4787-86e6-2943934f76ad
response_code = 200
uri = /vpc-routes
|
| 3114 | 2024-08-22T02:20:36.917Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:63125
|
| 3115 | 2024-08-22T02:20:36.917Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 50
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:63125
req_id = e41649e5-777e-4ebd-9386-498c4210226e
response_code = 200
uri = /vpc-routes
|
| 3116 | 2024-08-22T02:20:36.927Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 135
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:35316
req_id = 246ab31b-2aff-472f-aaf5-47dc3e65ae73
response_code = 204
uri = /vpc-routes
|
| 3117 | 2024-08-22T02:20:36.931Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1150
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:63125
req_id = bbe6908a-9fcf-4e63-9c64-47ecf298e04e
response_code = 204
uri = /vpc-routes
|
| 3118 | 2024-08-22T02:20:36.937Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:40001
|
| 3119 | 2024-08-22T02:20:36.937Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 137
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:40001
req_id = 7da6c4b8-5280-439e-b7ea-d83b92237134
response_code = 200
uri = /v2p
|
| 3120 | 2024-08-22T02:20:36.987Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:63599
|
| 3121 | 2024-08-22T02:20:36.987Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 113
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:63599
req_id = 4fc2fab3-193a-49f2-9eeb-9b615d166f12
response_code = 200
uri = /v2p
|
| 3122 | 2024-08-22T02:20:37.038Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:51924
|
| 3123 | 2024-08-22T02:20:37.038Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 49
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:51924
req_id = 43f791c5-2951-4937-8efa-e6f49016e181
response_code = 200
uri = /vpc-routes
|
| 3124 | 2024-08-22T02:20:37.047Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 74
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:51924
req_id = 95cd8e39-23e0-44b9-8bab-a35c16bd04eb
response_code = 204
uri = /vpc-routes
|
| 3125 | 2024-08-22T02:20:37.073Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:52054
|
| 3126 | 2024-08-22T02:20:37.074Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3127 | 2024-08-22T02:20:37.099Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3128 | 2024-08-22T02:20:37.125Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 51804
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:52054
req_id = d29c3fa2-9cf5-4511-a5ec-a043f350dd20
response_code = 204
uri = /switch-ports
|
| 3129 | 2024-08-22T02:20:37.125Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:60167
|
| 3130 | 2024-08-22T02:20:37.126Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3131 | 2024-08-22T02:20:37.149Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3132 | 2024-08-22T02:20:37.170Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 44903
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:60167
req_id = d333cff0-6f6d-48ba-9bf5-48c099b1dfe4
response_code = 204
uri = /switch-ports
|
| 3133 | 2024-08-22T02:20:37.171Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:59614
|
| 3134 | 2024-08-22T02:20:37.171Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 212
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:52054
req_id = 2299056c-e7cd-49a1-a2c1-8f274d2795e0
response_code = 200
uri = /network-bootstore-config
|
| 3135 | 2024-08-22T02:20:37.171Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 264
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:60167
req_id = 95c29049-91d4-4f69-b7a6-9218dde09231
response_code = 200
uri = /network-bootstore-config
|
| 3136 | 2024-08-22T02:20:37.172Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 846
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:59614
req_id = d45514cc-9245-420a-8374-5fcf9181b8ec
response_code = 200
uri = /v2p
|
| 3137 | 2024-08-22T02:20:37.241Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:56206
|
| 3138 | 2024-08-22T02:20:37.241Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3139 | 2024-08-22T02:20:37.266Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3140 | 2024-08-22T02:20:37.287Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 46033
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:56206
req_id = d797ec2e-2471-4c34-b79f-bae80a9b732e
response_code = 204
uri = /switch-ports
|
| 3141 | 2024-08-22T02:20:37.288Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 102
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:56206
req_id = 81162a38-b1aa-4930-b570-49ba4f663f09
response_code = 200
uri = /network-bootstore-config
|
| 3142 | 2024-08-22T02:20:42.773Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 6822
local_addr = [fd00:1122:3344:101::1]:42716
method = GET
remote_addr = [fd00:1122:3344:101::d]:54144
req_id = 2ccd4345-cf48-415e-87b4-feeadd229116
response_code = 200
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
uri = /37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3143 | 2024-08-22T02:20:46.055Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 3144 | 2024-08-22T02:20:51.846Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:48040
|
| 3145 | 2024-08-22T02:20:51.848Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 55
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:48040
req_id = f23654b1-cf9b-465a-b31f-57d91d9bf8ec
response_code = 200
uri = /baseboard
|
| 3146 | 2024-08-22T02:21:07.094Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:64412
|
| 3147 | 2024-08-22T02:21:07.095Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1385
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:64412
req_id = 33fb6ba3-7d2e-49f0-b4b2-01ea9a2497cd
response_code = 200
uri = /vpc-routes
|
| 3148 | 2024-08-22T02:21:07.104Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 102
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:64412
req_id = 8a0dbd3b-4903-4c42-a90e-6ff6dd9daaee
response_code = 204
uri = /vpc-routes
|
| 3149 | 2024-08-22T02:21:07.153Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:65319
|
| 3150 | 2024-08-22T02:21:07.153Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 42
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:65319
req_id = 82fdfbb7-6bb9-4e45-aa6c-0c5860a1efb0
response_code = 200
uri = /vpc-routes
|
| 3151 | 2024-08-22T02:21:07.161Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 75
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:65319
req_id = feca537a-a6de-4ee3-804e-2558517c5f1d
response_code = 204
uri = /vpc-routes
|
| 3152 | 2024-08-22T02:21:07.172Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:40811
|
| 3153 | 2024-08-22T02:21:07.172Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 34
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:40811
req_id = f9c94897-f6e7-4bd0-9cc9-ff1dcfd417dd
response_code = 200
uri = /vpc-routes
|
| 3154 | 2024-08-22T02:21:07.181Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 83
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:40811
req_id = c4a0dacd-f434-48c1-8106-1ee2b6996bfe
response_code = 204
uri = /vpc-routes
|
| 3155 | 2024-08-22T02:21:07.196Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:51040
|
| 3156 | 2024-08-22T02:21:07.198Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1152
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:51040
req_id = 8fee7186-4709-45db-869d-1f216bbc4777
response_code = 200
uri = /v2p
|
| 3157 | 2024-08-22T02:21:07.229Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:34373
|
| 3158 | 2024-08-22T02:21:07.229Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 115
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:34373
req_id = 3bf8e2f6-6b2d-423c-bff8-d294fef60705
response_code = 200
uri = /v2p
|
| 3159 | 2024-08-22T02:21:07.235Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:51358
|
| 3160 | 2024-08-22T02:21:07.236Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 78
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:51358
req_id = 5042d339-5ea6-4220-b76c-7deebde73291
response_code = 200
uri = /v2p
|
| 3161 | 2024-08-22T02:21:07.349Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:52931
|
| 3162 | 2024-08-22T02:21:07.349Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3163 | 2024-08-22T02:21:07.376Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3164 | 2024-08-22T02:21:07.398Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 48720
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:52931
req_id = bb64fd0b-299e-46e3-a793-fca9c343674f
response_code = 204
uri = /switch-ports
|
| 3165 | 2024-08-22T02:21:07.398Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:63408
|
| 3166 | 2024-08-22T02:21:07.398Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3167 | 2024-08-22T02:21:07.422Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3168 | 2024-08-22T02:21:07.444Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 45985
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:63408
req_id = f2b4f912-40e0-457b-bd43-2745e77574d5
response_code = 204
uri = /switch-ports
|
| 3169 | 2024-08-22T02:21:07.444Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:47268
|
| 3170 | 2024-08-22T02:21:07.444Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 145
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:52931
req_id = 60a6f79d-ecdb-4279-95eb-644584e229fa
response_code = 200
uri = /network-bootstore-config
|
| 3171 | 2024-08-22T02:21:07.444Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 266
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:63408
req_id = 7fef07c1-bc44-43cd-856d-e5f63c805767
response_code = 200
uri = /network-bootstore-config
|
| 3172 | 2024-08-22T02:21:07.444Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3173 | 2024-08-22T02:21:07.468Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3174 | 2024-08-22T02:21:07.492Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 47883
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:47268
req_id = d209eb39-f4fe-4bb3-b13e-612b25d155e7
response_code = 204
uri = /switch-ports
|
| 3175 | 2024-08-22T02:21:07.492Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 155
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:47268
req_id = 5a725c24-b8f9-43c2-a52d-69107dfe0023
response_code = 200
uri = /network-bootstore-config
|
| 3176 | 2024-08-22T02:21:12.895Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 6924
local_addr = [fd00:1122:3344:101::1]:42716
method = GET
remote_addr = [fd00:1122:3344:101::d]:54144
req_id = 9dee1d61-fdce-4948-bf33-4c0bc4476f38
response_code = 200
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
uri = /37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3177 | 2024-08-22T02:21:21.885Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:55738
|
| 3178 | 2024-08-22T02:21:21.885Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 57
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:55738
req_id = b3c48f18-d979-4a5f-9636-908649ab40c7
response_code = 200
uri = /baseboard
|
| 3179 | 2024-08-22T02:21:36.905Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:35330
|
| 3180 | 2024-08-22T02:21:36.905Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 128
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:35330
req_id = 78ada6fd-56f5-458b-b1f2-36fac8cf99fb
response_code = 200
uri = /vpc-routes
|
| 3181 | 2024-08-22T02:21:36.916Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 167
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:35330
req_id = ba9944a9-8376-40d0-9563-b6b9dc373fa3
response_code = 204
uri = /vpc-routes
|
| 3182 | 2024-08-22T02:21:36.929Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:48589
|
| 3183 | 2024-08-22T02:21:36.930Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 61
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:48589
req_id = bd5d4aea-d9a6-4ddf-824a-9374c9d578c9
response_code = 200
uri = /vpc-routes
|
| 3184 | 2024-08-22T02:21:36.946Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 140
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:48589
req_id = 40b96753-ef80-42bd-ba48-34787e06bf14
response_code = 204
uri = /vpc-routes
|
| 3185 | 2024-08-22T02:21:36.991Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:36257
|
| 3186 | 2024-08-22T02:21:36.992Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 174
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:36257
req_id = 1a775a4d-fa0f-488a-944f-9be9d83d33b6
response_code = 200
uri = /v2p
|
| 3187 | 2024-08-22T02:21:37.081Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:43101
|
| 3188 | 2024-08-22T02:21:37.081Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 158
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:43101
req_id = 8ee229a4-dfbd-425c-96b8-a0491d4f6395
response_code = 200
uri = /v2p
|
| 3189 | 2024-08-22T02:21:37.085Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:44817
|
| 3190 | 2024-08-22T02:21:37.085Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 72
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:44817
req_id = 9810c200-5a17-429b-add9-33d43ad8f4df
response_code = 200
uri = /v2p
|
| 3191 | 2024-08-22T02:21:37.092Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:38166
|
| 3192 | 2024-08-22T02:21:37.092Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3193 | 2024-08-22T02:21:37.119Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3194 | 2024-08-22T02:21:37.141Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 49404
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:38166
req_id = 2d608c34-bbb2-4f0b-b2cd-79c1b5b73418
response_code = 204
uri = /switch-ports
|
| 3195 | 2024-08-22T02:21:37.142Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 813
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:38166
req_id = c7f61973-e08f-4401-9ceb-9dedccc3c022
response_code = 200
uri = /network-bootstore-config
|
| 3196 | 2024-08-22T02:21:37.166Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:39777
|
| 3197 | 2024-08-22T02:21:37.166Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3198 | 2024-08-22T02:21:37.190Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3199 | 2024-08-22T02:21:37.212Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 46279
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:39777
req_id = 3babcfb8-7a28-4113-9601-9c9c3ddd51e3
response_code = 204
uri = /switch-ports
|
| 3200 | 2024-08-22T02:21:37.212Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:52123
|
| 3201 | 2024-08-22T02:21:37.212Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 91
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:52123
req_id = c039a9a4-e6a8-4860-aa45-ce60b9874a3a
response_code = 200
uri = /vpc-routes
|
| 3202 | 2024-08-22T02:21:37.212Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 187
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:39777
req_id = c600a4b8-9097-4f76-b0f9-807e6bd3c766
response_code = 200
uri = /network-bootstore-config
|
| 3203 | 2024-08-22T02:21:37.220Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 113
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:52123
req_id = 7d53ce16-d198-4a3d-8e6f-71e007946731
response_code = 204
uri = /vpc-routes
|
| 3204 | 2024-08-22T02:21:37.349Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:61184
|
| 3205 | 2024-08-22T02:21:37.349Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3206 | 2024-08-22T02:21:37.374Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3207 | 2024-08-22T02:21:37.396Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 47335
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:61184
req_id = 16c7fa93-d56b-4468-a787-ab10804666d6
response_code = 204
uri = /switch-ports
|
| 3208 | 2024-08-22T02:21:37.396Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 92
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:61184
req_id = c7f6a102-0f4d-40ec-9aa1-b9fa18ae3cac
response_code = 200
uri = /network-bootstore-config
|
| 3209 | 2024-08-22T02:21:42.895Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 6296
local_addr = [fd00:1122:3344:101::1]:42716
method = GET
remote_addr = [fd00:1122:3344:101::d]:54144
req_id = 95892912-3268-4afb-885e-c3f0f6a99419
response_code = 200
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
uri = /37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3210 | 2024-08-22T02:21:51.923Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56011
|
| 3211 | 2024-08-22T02:21:51.923Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 49
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56011
req_id = 72f3de4a-b3d0-4078-9842-3f45828e3c1b
response_code = 200
uri = /baseboard
|
| 3212 | 2024-08-22T02:22:06.970Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:38786
|
| 3213 | 2024-08-22T02:22:06.970Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 124
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:38786
req_id = 52911e53-bb70-4c2c-b660-0256fac93e65
response_code = 200
uri = /vpc-routes
|
| 3214 | 2024-08-22T02:22:06.979Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 157
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:38786
req_id = fb5ec75f-c0c4-4752-8e8e-b3d51d51b604
response_code = 204
uri = /vpc-routes
|
| 3215 | 2024-08-22T02:22:06.986Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:44725
|
| 3216 | 2024-08-22T02:22:06.986Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 40
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:44725
req_id = d85722eb-3210-433f-8e4e-6dfbfd0fcea0
response_code = 200
uri = /vpc-routes
|
| 3217 | 2024-08-22T02:22:06.996Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 84
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:44725
req_id = 5b9b33d1-7f41-488c-b0c9-706ff9a2a292
response_code = 204
uri = /vpc-routes
|
| 3218 | 2024-08-22T02:22:07.046Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:45246
|
| 3219 | 2024-08-22T02:22:07.047Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 151
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:45246
req_id = bde00b32-2f3a-42d9-8945-cadecb68a66a
response_code = 200
uri = /v2p
|
| 3220 | 2024-08-22T02:22:07.049Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:62050
|
| 3221 | 2024-08-22T02:22:07.049Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 165
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:62050
req_id = f33f66f5-bab7-4175-82f8-d49af738506b
response_code = 200
uri = /v2p
|
| 3222 | 2024-08-22T02:22:07.099Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:46404
|
| 3223 | 2024-08-22T02:22:07.099Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 51
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:46404
req_id = f6a4ddc1-94d4-4ccc-ab41-6cdf763f8ea6
response_code = 200
uri = /vpc-routes
|
| 3224 | 2024-08-22T02:22:07.108Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 99
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:46404
req_id = d36857ed-20b6-48ef-91ca-336c42b384cf
response_code = 204
uri = /vpc-routes
|
| 3225 | 2024-08-22T02:22:07.166Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:60054
|
| 3226 | 2024-08-22T02:22:07.166Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 126
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:60054
req_id = 8fed3edc-8116-46c3-b6c6-d144901d8589
response_code = 200
uri = /v2p
|
| 3227 | 2024-08-22T02:22:07.222Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:50913
|
| 3228 | 2024-08-22T02:22:07.223Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3229 | 2024-08-22T02:22:07.258Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3230 | 2024-08-22T02:22:07.289Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 67280
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:50913
req_id = fb49d606-11b9-49c6-9310-3e1821975009
response_code = 204
uri = /switch-ports
|
| 3231 | 2024-08-22T02:22:07.289Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:48731
|
| 3232 | 2024-08-22T02:22:07.290Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 287
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:50913
req_id = 906f263e-5c42-44c9-bd03-08597ca1884a
response_code = 200
uri = /network-bootstore-config
|
| 3233 | 2024-08-22T02:22:07.291Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3234 | 2024-08-22T02:22:07.321Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3235 | 2024-08-22T02:22:07.350Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 60002
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:48731
req_id = e7e5a1e4-4761-4387-90ed-71c2b7b29af8
response_code = 204
uri = /switch-ports
|
| 3236 | 2024-08-22T02:22:07.350Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 155
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:48731
req_id = c3a0241b-592c-4d7e-9ce3-3edc30d453f7
response_code = 200
uri = /network-bootstore-config
|
| 3237 | 2024-08-22T02:22:07.472Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:32903
|
| 3238 | 2024-08-22T02:22:07.473Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3239 | 2024-08-22T02:22:07.498Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3240 | 2024-08-22T02:22:07.520Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 47401
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:32903
req_id = 272d92b7-354f-4f03-b7f0-1cfe8fb02905
response_code = 204
uri = /switch-ports
|
| 3241 | 2024-08-22T02:22:07.520Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 109
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:32903
req_id = fe868be1-0513-450d-8d10-48dfba38b117
response_code = 200
uri = /network-bootstore-config
|
| 3242 | 2024-08-22T02:22:12.895Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 6337
local_addr = [fd00:1122:3344:101::1]:42716
method = GET
remote_addr = [fd00:1122:3344:101::d]:54144
req_id = 87964dbd-18a7-4695-a67e-fc1e065d3d2f
response_code = 200
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
uri = /37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3243 | 2024-08-22T02:22:21.957Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:47872
|
| 3244 | 2024-08-22T02:22:21.957Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 84
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:47872
req_id = a7763dd6-c034-40d4-989b-41fbdaa3bc55
response_code = 200
uri = /baseboard
|
| 3245 | 2024-08-22T02:22:36.914Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:62849
|
| 3246 | 2024-08-22T02:22:36.915Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 985
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:62849
req_id = c304cc7c-952d-410d-b088-6df87d327f44
response_code = 200
uri = /vpc-routes
|
| 3247 | 2024-08-22T02:22:36.926Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1380
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:62849
req_id = c354f175-322c-4cc3-bce0-a12ee03ca0c3
response_code = 204
uri = /vpc-routes
|
| 3248 | 2024-08-22T02:22:36.949Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:60150
|
| 3249 | 2024-08-22T02:22:36.949Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 113
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:60150
req_id = 1aada8b4-145c-4804-b5ab-1ca8328022d2
response_code = 200
uri = /v2p
|
| 3250 | 2024-08-22T02:22:37.024Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:46674
|
| 3251 | 2024-08-22T02:22:37.024Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 48
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:46674
req_id = eced51e3-ea34-4b88-929c-424d74088f44
response_code = 200
uri = /vpc-routes
|
| 3252 | 2024-08-22T02:22:37.032Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 78
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:46674
req_id = 9cf84964-4c5d-469b-ad2e-9231d1ab3db3
response_code = 204
uri = /vpc-routes
|
| 3253 | 2024-08-22T02:22:37.047Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:55687
|
| 3254 | 2024-08-22T02:22:37.047Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 92
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:55687
req_id = a07e811b-95d9-4444-9fd8-9f480091553f
response_code = 200
uri = /v2p
|
| 3255 | 2024-08-22T02:22:37.083Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:45962
|
| 3256 | 2024-08-22T02:22:37.083Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 719
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:45962
req_id = 9cb1511e-3fcb-4b0c-beb8-eac98169f1a3
response_code = 200
uri = /v2p
|
| 3257 | 2024-08-22T02:22:37.136Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:32899
|
| 3258 | 2024-08-22T02:22:37.136Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3259 | 2024-08-22T02:22:37.162Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3260 | 2024-08-22T02:22:37.186Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 49657
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:32899
req_id = 6366f6e7-6e25-4346-a8f2-7ee9c9c1ce90
response_code = 204
uri = /switch-ports
|
| 3261 | 2024-08-22T02:22:37.186Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:47414
|
| 3262 | 2024-08-22T02:22:37.186Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:54452
|
| 3263 | 2024-08-22T02:22:37.186Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3264 | 2024-08-22T02:22:37.190Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 3576
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:32899
req_id = 4de01409-ad9b-4aea-a0cd-bb8ef253d00c
response_code = 200
uri = /network-bootstore-config
|
| 3265 | 2024-08-22T02:22:37.190Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 3682
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:54452
req_id = 9de68e2d-4a42-41c4-a6ce-02076b239255
response_code = 200
uri = /vpc-routes
|
| 3266 | 2024-08-22T02:22:37.197Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 165
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:54452
req_id = bb5ac7fa-da2d-4ac5-91b2-4c6273366787
response_code = 204
uri = /vpc-routes
|
| 3267 | 2024-08-22T02:22:37.212Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3268 | 2024-08-22T02:22:37.233Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 47668
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:47414
req_id = 7280d410-413e-440d-a122-12de3f91e377
response_code = 204
uri = /switch-ports
|
| 3269 | 2024-08-22T02:22:37.235Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 879
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:47414
req_id = 7efd4d44-efd6-462c-95e9-5e301407a4e2
response_code = 200
uri = /network-bootstore-config
|
| 3270 | 2024-08-22T02:22:37.244Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:52692
|
| 3271 | 2024-08-22T02:22:37.244Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3272 | 2024-08-22T02:22:37.267Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3273 | 2024-08-22T02:22:37.290Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 46109
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:52692
req_id = 8e60e278-4c16-4786-9297-a94ee0d57005
response_code = 204
uri = /switch-ports
|
| 3274 | 2024-08-22T02:22:37.290Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 94
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:52692
req_id = abf0cc96-025f-4998-bfdd-3f2b423c71df
response_code = 200
uri = /network-bootstore-config
|
| 3275 | 2024-08-22T02:22:42.896Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 7016
local_addr = [fd00:1122:3344:101::1]:42716
method = GET
remote_addr = [fd00:1122:3344:101::d]:54144
req_id = 7adbc27b-ff04-4f0f-ba99-fcde9d4f9772
response_code = 200
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
uri = /37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3276 | 2024-08-22T02:22:51.992Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:62128
|
| 3277 | 2024-08-22T02:22:51.992Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 39
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:62128
req_id = 9052c01e-94d8-4c09-956e-5dd498336fc2
response_code = 200
uri = /baseboard
|
| 3278 | 2024-08-22T02:23:07.117Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:37271
|
| 3279 | 2024-08-22T02:23:07.117Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 196
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:37271
req_id = 16fdea25-c2ec-465d-8a73-ddb34b217c3b
response_code = 200
uri = /v2p
|
| 3280 | 2024-08-22T02:23:07.171Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:51685
|
| 3281 | 2024-08-22T02:23:07.171Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 113
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:51685
req_id = 1127d66e-bc1e-4d32-a37c-ccc711269e35
response_code = 200
uri = /v2p
|
| 3282 | 2024-08-22T02:23:07.209Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:35377
|
| 3283 | 2024-08-22T02:23:07.209Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 72
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:35377
req_id = 74b98c14-801f-44d4-833e-73305ce4d1bb
response_code = 200
uri = /vpc-routes
|
| 3284 | 2024-08-22T02:23:07.218Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 117
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:35377
req_id = abdc3dde-3f0f-466e-aa03-7793c7474982
response_code = 204
uri = /vpc-routes
|
| 3285 | 2024-08-22T02:23:07.223Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:59956
|
| 3286 | 2024-08-22T02:23:07.223Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:671
rules = [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 3287 | 2024-08-22T02:23:07.223Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 3288 | 2024-08-22T02:23:07.223Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 3289 | 2024-08-22T02:23:07.223Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 3290 | 2024-08-22T02:23:07.223Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte0"
rules = []
|
| 3291 | 2024-08-22T02:23:07.223Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 3292 | 2024-08-22T02:23:07.223Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 3293 | 2024-08-22T02:23:07.223Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 281
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:59956
req_id = 60546535-9c82-4590-bc28-b71285c49560
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 3294 | 2024-08-22T02:23:07.266Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:62372
|
| 3295 | 2024-08-22T02:23:07.267Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1282
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:62372
req_id = 20b33b84-7729-48ed-bff0-7c0d49da7952
response_code = 200
uri = /vpc-routes
|
| 3296 | 2024-08-22T02:23:07.275Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 88
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:62372
req_id = 6b25ffa8-ff7e-428d-9181-407621504742
response_code = 204
uri = /vpc-routes
|
| 3297 | 2024-08-22T02:23:07.297Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:47991
|
| 3298 | 2024-08-22T02:23:07.298Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1371
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:47991
req_id = 1557c10f-6a20-4606-aa75-162e79db91ab
response_code = 200
uri = /vpc-routes
|
| 3299 | 2024-08-22T02:23:07.306Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 75
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:47991
req_id = 7332d521-72a5-41fa-bffb-8c9b9d3d3ffc
response_code = 204
uri = /vpc-routes
|
| 3300 | 2024-08-22T02:23:07.319Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:57557
|
| 3301 | 2024-08-22T02:23:07.319Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 102
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:57557
req_id = de6fecb1-154e-4397-813b-9ee190edcd35
response_code = 200
uri = /v2p
|
| 3302 | 2024-08-22T02:23:07.407Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:64698
|
| 3303 | 2024-08-22T02:23:07.407Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:671
rules = [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 3304 | 2024-08-22T02:23:07.407Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 3305 | 2024-08-22T02:23:07.407Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 3306 | 2024-08-22T02:23:07.407Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 3307 | 2024-08-22T02:23:07.407Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte0"
rules = []
|
| 3308 | 2024-08-22T02:23:07.407Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 3309 | 2024-08-22T02:23:07.407Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 3310 | 2024-08-22T02:23:07.407Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 279
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:64698
req_id = dfa478b0-0387-4523-87ba-b6eb4f69c585
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 3311 | 2024-08-22T02:23:07.498Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:36990
|
| 3312 | 2024-08-22T02:23:07.498Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3313 | 2024-08-22T02:23:07.525Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3314 | 2024-08-22T02:23:07.549Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 51548
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:36990
req_id = 8da8d4cb-4035-479f-84ca-5c34fc1e4be3
response_code = 204
uri = /switch-ports
|
| 3315 | 2024-08-22T02:23:07.549Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:43339
|
| 3316 | 2024-08-22T02:23:07.550Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:671
rules = [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 13ec8b2e-313a-4409-8282-7e65d573a5d1, kind: Service { id: 7d0db839-e1b4-4b99-8b58-7f70a3941e8d }, name: Name("external-dns-7d0db839-e1b4-4b99-8b58-7f70a3941e8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 221])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a56ed08e-f6f5-42f9-8676-08c27c1c212e, kind: Service { id: e3cfd6ed-225d-4c9e-a437-60c94e80df50 }, name: Name("external-dns-e3cfd6ed-225d-4c9e-a437-60c94e80df50"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 98])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 34375107-1ce1-4762-9001-68b7ab7f0c68, kind: Service { id: 7524dcb2-fbae-4cac-83ef-d3e4411c247f }, name: Name("nexus-7524dcb2-fbae-4cac-83ef-d3e4411c247f"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 202])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 71b9db92-c609-4aed-802a-821326a72811, kind: Service { id: 249b0525-e315-49f0-bea0-213c52bdfd49 }, name: Name("nexus-249b0525-e315-49f0-bea0-213c52bdfd49"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e46f8a5b-c887-4ee4-b6ce-42e4d5b5bf08, kind: Service { id: 95247a59-463e-4d62-a4ab-f35b1fbfb959 }, name: Name("nexus-95247a59-463e-4d62-a4ab-f35b1fbfb959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 61])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 3317 | 2024-08-22T02:23:07.550Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 3318 | 2024-08-22T02:23:07.550Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 255
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:36990
req_id = a6fc638c-e456-4e1b-a370-9e773128a74d
response_code = 200
uri = /network-bootstore-config
|
| 3319 | 2024-08-22T02:23:07.550Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 3320 | 2024-08-22T02:23:07.550Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 3321 | 2024-08-22T02:23:07.550Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte0"
rules = []
|
| 3322 | 2024-08-22T02:23:07.550Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 3323 | 2024-08-22T02:23:07.550Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:690
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 3324 | 2024-08-22T02:23:07.550Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 482
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:43339
req_id = 347c29d1-b5d6-461d-803d-4e9189a501ca
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 3325 | 2024-08-22T02:23:07.572Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:48462
|
| 3326 | 2024-08-22T02:23:07.572Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3327 | 2024-08-22T02:23:07.595Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3328 | 2024-08-22T02:23:07.618Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 46094
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:48462
req_id = b6f08ade-2dd8-41d1-8d4c-c20abd6a7ec1
response_code = 204
uri = /switch-ports
|
| 3329 | 2024-08-22T02:23:07.618Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 244
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:48462
req_id = 1c521fed-aa00-4bb9-bea1-a6eeb954bf05
response_code = 200
uri = /network-bootstore-config
|
| 3330 | 2024-08-22T02:23:07.640Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:50792
|
| 3331 | 2024-08-22T02:23:07.640Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3332 | 2024-08-22T02:23:07.663Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3333 | 2024-08-22T02:23:07.683Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 42531
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:50792
req_id = 892a1f78-5e4a-4d0f-ae44-713b87f61dec
response_code = 204
uri = /switch-ports
|
| 3334 | 2024-08-22T02:23:07.683Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 98
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:50792
req_id = 1d23cf1b-43ad-46a0-b86f-b329eec40c9e
response_code = 200
uri = /network-bootstore-config
|
| 3335 | 2024-08-22T02:23:12.896Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 6195
local_addr = [fd00:1122:3344:101::1]:42716
method = GET
remote_addr = [fd00:1122:3344:101::d]:54144
req_id = 9b33da3b-eb78-4315-8a79-4c475b5e303a
response_code = 200
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
uri = /37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3336 | 2024-08-22T02:23:22.026Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:54082
|
| 3337 | 2024-08-22T02:23:22.027Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 675
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:54082
req_id = a4b5ebc8-165d-4bdc-9f41-347161c7c992
response_code = 200
uri = /baseboard
|
| 3338 | 2024-08-22T02:23:36.920Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:53153
|
| 3339 | 2024-08-22T02:23:36.921Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 991
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:53153
req_id = 8b99e6fc-642b-44fd-9f31-13950e348fb7
response_code = 200
uri = /vpc-routes
|
| 3340 | 2024-08-22T02:23:36.928Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 133
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:53153
req_id = 62fd001d-1971-41f7-b403-021853793c73
response_code = 204
uri = /vpc-routes
|
| 3341 | 2024-08-22T02:23:36.943Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:51595
|
| 3342 | 2024-08-22T02:23:36.943Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 33
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:51595
req_id = afe1271b-611d-4172-89e6-d1ac2ef5b346
response_code = 200
uri = /vpc-routes
|
| 3343 | 2024-08-22T02:23:36.945Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:36261
|
| 3344 | 2024-08-22T02:23:36.945Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 106
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:36261
req_id = 0d8ad4a2-c99d-492b-9ea2-6a0f00a9871b
response_code = 200
uri = /v2p
|
| 3345 | 2024-08-22T02:23:36.950Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 56
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:51595
req_id = 117c3f74-cd88-4eae-83de-1f7fdf52740d
response_code = 204
uri = /vpc-routes
|
| 3346 | 2024-08-22T02:23:36.957Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:42734
|
| 3347 | 2024-08-22T02:23:36.958Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 907
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:42734
req_id = 4de21ddd-26c3-4bfa-be95-9439d1ff49c2
response_code = 200
uri = /v2p
|
| 3348 | 2024-08-22T02:23:37.032Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:61507
|
| 3349 | 2024-08-22T02:23:37.033Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 665
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:61507
req_id = e03bab91-5ff7-4f1b-b6d3-9857cee9a366
response_code = 200
uri = /vpc-routes
|
| 3350 | 2024-08-22T02:23:37.040Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 67
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:61507
req_id = f7d296cc-fa6f-44b2-a1fb-4a333df39088
response_code = 204
uri = /vpc-routes
|
| 3351 | 2024-08-22T02:23:37.049Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:52107
|
| 3352 | 2024-08-22T02:23:37.050Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 71
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:52107
req_id = cdfbaa69-04fc-426c-b540-aef6813a5d25
response_code = 200
uri = /v2p
|
| 3353 | 2024-08-22T02:23:37.087Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:42574
|
| 3354 | 2024-08-22T02:23:37.087Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3355 | 2024-08-22T02:23:37.112Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3356 | 2024-08-22T02:23:37.133Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 45922
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:42574
req_id = 69011abc-dd46-4d8e-b61e-b79c03654219
response_code = 204
uri = /switch-ports
|
| 3357 | 2024-08-22T02:23:37.133Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 96
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:42574
req_id = 74ed0b56-4fa9-4ea9-85ff-69db1bf7f31a
response_code = 200
uri = /network-bootstore-config
|
| 3358 | 2024-08-22T02:23:37.185Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:57824
|
| 3359 | 2024-08-22T02:23:37.185Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3360 | 2024-08-22T02:23:37.209Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3361 | 2024-08-22T02:23:37.232Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 47546
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:57824
req_id = 89f5ca29-385f-4d1b-807e-a1ae322b699c
response_code = 204
uri = /switch-ports
|
| 3362 | 2024-08-22T02:23:37.234Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 894
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:57824
req_id = a72d2689-2aa0-4241-b5ab-cd7015d1ad4f
response_code = 200
uri = /network-bootstore-config
|
| 3363 | 2024-08-22T02:23:37.360Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:40000
|
| 3364 | 2024-08-22T02:23:37.361Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3365 | 2024-08-22T02:23:37.386Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3366 | 2024-08-22T02:23:37.407Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 46155
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:40000
req_id = c5a384f1-42c8-4099-a0cf-0817c1e4f5f7
response_code = 204
uri = /switch-ports
|
| 3367 | 2024-08-22T02:23:37.407Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 88
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:40000
req_id = fa0ad86b-fcbb-43a1-b98c-efa22635b2b5
response_code = 200
uri = /network-bootstore-config
|
| 3368 | 2024-08-22T02:23:42.978Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 6853
local_addr = [fd00:1122:3344:101::1]:42716
method = GET
remote_addr = [fd00:1122:3344:101::d]:54144
req_id = d6b20828-beb2-406a-a763-15a03494dc86
response_code = 200
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
uri = /37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3369 | 2024-08-22T02:23:46.028Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:57184
|
| 3370 | 2024-08-22T02:23:46.028Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 154
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:57184
req_id = b94f52d4-738d-4373-bad4-2296702029f0
response_code = 200
uri = /v2p
|
| 3371 | 2024-08-22T02:23:46.029Z | INFO | SledAgent (PortManager): Mapping virtual NIC to physical host
file = illumos-utils/src/opte/port_manager.rs:778
mapping = VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 244, 216, 235])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(11127658) }
|
| 3372 | 2024-08-22T02:23:46.029Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 780
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:57184
req_id = ed5dd810-a650-4858-bb91-86445864670e
response_code = 204
uri = /v2p
|
| 3373 | 2024-08-22T02:23:46.123Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:42737
|
| 3374 | 2024-08-22T02:23:46.134Z | INFO | SledAgent (InstanceManager): ensuring instance is registered
file = sled-agent/src/instance_manager.rs:569
hardware = InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname("localshark") }, nics: [NetworkInterface { id: 12a669f9-d1a7-46ce-82ae-dd504675fd2c, kind: Instance { id: 4b73965e-c980-4acb-a19e-b90a84025ee8 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 216, 235])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11127658), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 12a669f9-d1a7-46ce-82ae-dd504675fd2c, kind: Instance { id: 4b73965e-c980-4acb-a19e-b90a84025ee8 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 216, 235])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11127658), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 12a669f9-d1a7-46ce-82ae-dd504675fd2c, kind: Instance { id: 4b73965e-c980-4acb-a19e-b90a84025ee8 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 216, 235])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11127658), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(11127658))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 12a669f9-d1a7-46ce-82ae-dd504675fd2c, kind: Instance { id: 4b73965e-c980-4acb-a19e-b90a84025ee8 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 216, 235])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11127658), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: "nvme", name: "disk-e6f023ec147", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 965aa7db-25aa-4a23-a74b-92de2b3392ed, read_only_parent: Some(Volume { block_size: 512, id: 6103859b-8047-447c-bf6a-34b3b43f06b9, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 161c179d-f379-4b79-920a-5e80dcdb35a8, key: Some("guddHD4fxdtSKrEApmOSlvMlCdFYOAg1OxIvSytmwG0="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::12]:19002", "[fd00:1122:3344:101::14]:19001", "[fd00:1122:3344:101::18]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 965aa7db-25aa-4a23-a74b-92de2b3392ed, key: Some("mKFtuCW8OgDo1DMkRYJeOwXAqnigvscfzfJ6gemdgXQ="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::19]:19001", "[fd00:1122:3344:101::14]:19002", "[fd00:1122:3344:101::12]:19003"] } }] } }], cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAI9xIWWRZZAAD3EhZZAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAACPcSFlkWWQAA9xIWWQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0YjczOTY1ZS1jOTgwLTRhY2ItYTE5ZS1iOTBhODQwMjVlZTgiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRmlOWHROaG1mL3RJVVlMVHVUd0NCcGhnbXZXOHJQdnlWWHo1bGl2cjBoMyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") }
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
instance_runtime = InstanceRuntimeState { propolis_id: Some(20287db9-9bf9-4ca1-a2ef-9f74c529a27f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-08-22T02:23:45.790577660Z }
metadata = InstanceMetadata { silo_id: 428b7cfb-889d-459c-a33c-8042d31d2c46, project_id: d0211e3b-bc22-4e8a-87ad-077bf109afec }
propolis_addr = [fd00:1122:3344:101::1:0]:12400
propolis_id = 20287db9-9bf9-4ca1-a2ef-9f74c529a27f
vmm_runtime = VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-08-22T02:23:45.755134Z }
|
| 3375 | 2024-08-22T02:23:46.134Z | INFO | SledAgent (InstanceManager): registering new instance
file = sled-agent/src/instance_manager.rs:603
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8 (instance)
|
| 3376 | 2024-08-22T02:23:46.134Z | INFO | SledAgent (InstanceManager): initializing new Instance
file = sled-agent/src/instance.rs:994
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
migration_id = None
propolis_id = 20287db9-9bf9-4ca1-a2ef-9f74c529a27f
state = InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname("localshark") }, nics: [NetworkInterface { id: 12a669f9-d1a7-46ce-82ae-dd504675fd2c, kind: Instance { id: 4b73965e-c980-4acb-a19e-b90a84025ee8 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 216, 235])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11127658), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 12a669f9-d1a7-46ce-82ae-dd504675fd2c, kind: Instance { id: 4b73965e-c980-4acb-a19e-b90a84025ee8 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 216, 235])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11127658), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 12a669f9-d1a7-46ce-82ae-dd504675fd2c, kind: Instance { id: 4b73965e-c980-4acb-a19e-b90a84025ee8 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 216, 235])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11127658), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(11127658))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 12a669f9-d1a7-46ce-82ae-dd504675fd2c, kind: Instance { id: 4b73965e-c980-4acb-a19e-b90a84025ee8 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 216, 235])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11127658), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: "nvme", name: "disk-e6f023ec147", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 965aa7db-25aa-4a23-a74b-92de2b3392ed, read_only_parent: Some(Volume { block_size: 512, id: 6103859b-8047-447c-bf6a-34b3b43f06b9, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 161c179d-f379-4b79-920a-5e80dcdb35a8, key: Some("guddHD4fxdtSKrEApmOSlvMlCdFYOAg1OxIvSytmwG0="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::12]:19002", "[fd00:1122:3344:101::14]:19001", "[fd00:1122:3344:101::18]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 965aa7db-25aa-4a23-a74b-92de2b3392ed, key: Some("mKFtuCW8OgDo1DMkRYJeOwXAqnigvscfzfJ6gemdgXQ="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::19]:19001", "[fd00:1122:3344:101::14]:19002", "[fd00:1122:3344:101::12]:19003"] } }] } }], cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAI9xIWWRZZAAD3EhZZAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAACPcSFlkWWQAA9xIWWQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0YjczOTY1ZS1jOTgwLTRhY2ItYTE5ZS1iOTBhODQwMjVlZTgiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRmlOWHROaG1mL3RJVVlMVHVUd0NCcGhnbXZXOHJQdnlWWHo1bGl2cjBoMyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-08-22T02:23:45.755134Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }
|
| 3377 | 2024-08-22T02:23:46.134Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 11129
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:42737
req_id = c6264eb6-e13f-407d-9ade-d5a684705637
response_code = 200
uri = /instances/4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3378 | 2024-08-22T02:23:46.228Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:35056
|
| 3379 | 2024-08-22T02:23:46.231Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 3234
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:35056
req_id = b2ce2c08-c0f4-4d65-be1e-d24e7fefd146
response_code = 200
uri = /instances/4b73965e-c980-4acb-a19e-b90a84025ee8/state
|
| 3380 | 2024-08-22T02:23:46.236Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:424
port = Port { inner: PortInner(PortData { name: "opte6", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 244, 216, 235]), slot: 0, vni: Vni { inner: 11127658 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }
|
| 3381 | 2024-08-22T02:23:46.237Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 3382 | 2024-08-22T02:23:46.271Z | INFO | SledAgent (InstanceManager): Configuring new Omicron zone: oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f
file = illumos-utils/src/zone.rs:316
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3383 | 2024-08-22T02:23:46.292Z | INFO | SledAgent (InstanceManager): Installing Omicron zone: oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f
file = illumos-utils/src/zone.rs:354
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3384 | 2024-08-22T02:23:47.605Z | INFO | SledAgent (InstanceManager): Profile for oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="system/illumos/propolis-server">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlInstance0'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::1:0'/>
<propval type="astring" name="listen_port" value='12400'/>
<propval type="astring" name="metric_addr" value='dns'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3385 | 2024-08-22T02:23:47.605Z | INFO | SledAgent (InstanceManager): Booting oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f zone
file = illumos-utils/src/running_zone.rs:488
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
zone = oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f
|
| 3386 | 2024-08-22T02:23:47.780Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f") failed: Property not found. retry in 25.189532ms
file = illumos-utils/src/svc.rs:36
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
zone = oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f
|
| 3387 | 2024-08-22T02:23:47.814Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f") failed: Property not found. retry in 65.715229ms
file = illumos-utils/src/svc.rs:36
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
zone = oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f
|
| 3388 | 2024-08-22T02:23:47.896Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f") failed: Property not found. retry in 284.537166ms
file = illumos-utils/src/svc.rs:36
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
zone = oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f
|
| 3389 | 2024-08-22T02:23:48.195Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f") failed: Property not found. retry in 408.011659ms
file = illumos-utils/src/svc.rs:36
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
zone = oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f
|
| 3390 | 2024-08-22T02:23:48.613Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f") failed: Property not found. retry in 1.028977488s
file = illumos-utils/src/svc.rs:36
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
zone = oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f
|
| 3391 | 2024-08-22T02:23:49.655Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f") failed: Property not found. retry in 630.458821ms
file = illumos-utils/src/svc.rs:36
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
zone = oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f
|
| 3392 | 2024-08-22T02:23:50.296Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f") failed: Property not found. retry in 1.369134454s
file = illumos-utils/src/svc.rs:36
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
zone = oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f
|
| 3393 | 2024-08-22T02:23:51.675Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f") failed: Property not found. retry in 1.156950297s
file = illumos-utils/src/svc.rs:36
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
zone = oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f
|
| 3394 | 2024-08-22T02:23:52.060Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:35258
|
| 3395 | 2024-08-22T02:23:52.062Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 744
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:35258
req_id = b36a18fe-e9d7-48f7-b2e4-f61761f6de85
response_code = 200
uri = /baseboard
|
| 3396 | 2024-08-22T02:23:52.861Z | INFO | SledAgent (InstanceManager): Started propolis in zone: oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f
file = sled-agent/src/instance.rs:1448
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3397 | 2024-08-22T02:23:52.870Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f") failed: Property not found. retry in 58.021755ms
file = illumos-utils/src/svc.rs:36
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3398 | 2024-08-22T02:23:52.938Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f") failed: Property not found. retry in 66.083568ms
file = illumos-utils/src/svc.rs:36
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3399 | 2024-08-22T02:23:53.013Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f") failed: Property not found. retry in 203.00344ms
file = illumos-utils/src/svc.rs:36
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3400 | 2024-08-22T02:23:53.228Z | INFO | SledAgent (InstanceManager): Propolis SMF service is online
file = sled-agent/src/instance.rs:1457
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3401 | 2024-08-22T02:23:53.268Z | INFO | SledAgent (InstanceManager): Propolis HTTP server online
file = sled-agent/src/instance.rs:1481
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3402 | 2024-08-22T02:23:53.268Z | INFO | SledAgent (InstanceManager): Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAI9xIWWRZZAAD3EhZZAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAACPcSFlkWWQAA9xIWWQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0YjczOTY1ZS1jOTgwLTRhY2ItYTE5ZS1iOTBhODQwMjVlZTgiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRmlOWHROaG1mL3RJVVlMVHVUd0NCcGhnbXZXOHJQdnlWWHo1bGl2cjBoMyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"), disks: [DiskRequest { device: "nvme", name: "disk-e6f023ec147", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 965aa7db-25aa-4a23-a74b-92de2b3392ed, read_only_parent: Some(Volume { block_size: 512, id: 6103859b-8047-447c-bf6a-34b3b43f06b9, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 161c179d-f379-4b79-920a-5e80dcdb35a8, key: Some("guddHD4fxdtSKrEApmOSlvMlCdFYOAg1OxIvSytmwG0="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::12]:19002", "[fd00:1122:3344:101::14]:19001", "[fd00:1122:3344:101::18]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 965aa7db-25aa-4a23-a74b-92de2b3392ed, key: Some("mKFtuCW8OgDo1DMkRYJeOwXAqnigvscfzfJ6gemdgXQ="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::19]:19001", "[fd00:1122:3344:101::14]:19002", "[fd00:1122:3344:101::12]:19003"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: "opte6", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: "Test description", id: 4b73965e-c980-4acb-a19e-b90a84025ee8, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: d0211e3b-bc22-4e8a-87ad-077bf109afec, silo_id: 428b7cfb-889d-459c-a33c-8042d31d2c46, sled_id: 37fb4e95-8c94-4aa8-946d-de6b4770caf9, sled_model: "i86pc", sled_revision: 0, sled_serial: "sock" }, name: "localshark", vcpus: 2 } }
file = sled-agent/src/instance.rs:730
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3403 | 2024-08-22T02:23:53.423Z | INFO | SledAgent (InstanceManager): result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })
file = sled-agent/src/instance.rs:732
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3404 | 2024-08-22T02:23:53.424Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-08-22T02:23:53.424932313Z }
file = sled-agent/src/instance.rs:600
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3405 | 2024-08-22T02:23:53.424Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:622
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
new_vmm_state = VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-08-22T02:23:53.424932313Z }
propolis_id = 20287db9-9bf9-4ca1-a2ef-9f74c529a27f
|
| 3406 | 2024-08-22T02:23:53.425Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:520
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
state = SledInstanceState { propolis_id: 20287db9-9bf9-4ca1-a2ef-9f74c529a27f (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-08-22T02:23:53.424932313Z }, migration_in: None, migration_out: None }
|
| 3407 | 2024-08-22T02:23:53.468Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-08-22T02:23:53.468068971Z }
file = sled-agent/src/instance.rs:600
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3408 | 2024-08-22T02:23:53.468Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:622
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
new_vmm_state = VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-08-22T02:23:53.468068971Z }
propolis_id = 20287db9-9bf9-4ca1-a2ef-9f74c529a27f
|
| 3409 | 2024-08-22T02:23:53.468Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:520
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
state = SledInstanceState { propolis_id: 20287db9-9bf9-4ca1-a2ef-9f74c529a27f (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-08-22T02:23:53.468068971Z }, migration_in: None, migration_out: None }
|
| 3410 | 2024-08-22T02:24:07.045Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:36568
|
| 3411 | 2024-08-22T02:24:07.045Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 207
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:36568
req_id = 25e47322-ac83-4c0a-aa47-97544c624aba
response_code = 200
uri = /v2p
|
| 3412 | 2024-08-22T02:24:07.062Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:51685
|
| 3413 | 2024-08-22T02:24:07.062Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 260
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:51685
req_id = a3c69cba-cc50-493e-a09b-15ca3b6598de
response_code = 200
uri = /instances/4b73965e-c980-4acb-a19e-b90a84025ee8/state
|
| 3414 | 2024-08-22T02:24:07.074Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:36490
|
| 3415 | 2024-08-22T02:24:07.076Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1231
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:36490
req_id = 6b860a70-4cdf-4f71-b5d9-57893e524046
response_code = 200
uri = /vpc-routes
|
| 3416 | 2024-08-22T02:24:07.088Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:43179
|
| 3417 | 2024-08-22T02:24:07.088Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 104
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:43179
req_id = c3ee6362-3ca7-4069-8449-696801b87fb3
response_code = 200
uri = /instances/4b73965e-c980-4acb-a19e-b90a84025ee8/state
|
| 3418 | 2024-08-22T02:24:07.139Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 1453
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:36490
req_id = 4e4bca1f-749d-483d-a773-9c6496235706
response_code = 204
uri = /vpc-routes
|
| 3419 | 2024-08-22T02:24:07.194Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:53791
|
| 3420 | 2024-08-22T02:24:07.195Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 45
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:53791
req_id = 862846e3-f0d1-4f65-96ac-51087d6db0e7
response_code = 200
uri = /vpc-routes
|
| 3421 | 2024-08-22T02:24:07.209Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 90
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:53791
req_id = addc9d20-c6c1-450b-acf4-1b2052d25469
response_code = 204
uri = /vpc-routes
|
| 3422 | 2024-08-22T02:24:07.246Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:45734
|
| 3423 | 2024-08-22T02:24:07.246Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 112
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:45734
req_id = ab84bb52-427d-4c56-a21e-1e54c5e9ae5a
response_code = 200
uri = /v2p
|
| 3424 | 2024-08-22T02:24:07.293Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:53571
|
| 3425 | 2024-08-22T02:24:07.293Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 36
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:53571
req_id = ff18a1c5-d010-49c1-a870-be682cace82d
response_code = 200
uri = /vpc-routes
|
| 3426 | 2024-08-22T02:24:07.309Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 107
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:53571
req_id = ab0777de-3ec3-43e3-b1b3-ee18dd98dcd0
response_code = 204
uri = /vpc-routes
|
| 3427 | 2024-08-22T02:24:07.334Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:60120
|
| 3428 | 2024-08-22T02:24:07.334Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3429 | 2024-08-22T02:24:07.360Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3430 | 2024-08-22T02:24:07.383Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 48797
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:60120
req_id = 68562813-e1a0-44de-a49b-58790f3760a2
response_code = 204
uri = /switch-ports
|
| 3431 | 2024-08-22T02:24:07.383Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:62307
|
| 3432 | 2024-08-22T02:24:07.383Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:39092
|
| 3433 | 2024-08-22T02:24:07.383Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:38471
|
| 3434 | 2024-08-22T02:24:07.383Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3435 | 2024-08-22T02:24:07.388Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 5195
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:62307
req_id = 70aae147-2300-4b18-b981-4006d9ce2de5
response_code = 200
uri = /v2p
|
| 3436 | 2024-08-22T02:24:07.388Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 786
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:60120
req_id = fb677415-2fd9-40e0-91a6-355c63791d56
response_code = 200
uri = /network-bootstore-config
|
| 3437 | 2024-08-22T02:24:07.388Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 5436
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:38471
req_id = 5e0e99c0-d55b-4040-af59-fe45b3e01761
response_code = 200
uri = /instances/4b73965e-c980-4acb-a19e-b90a84025ee8/state
|
| 3438 | 2024-08-22T02:24:07.408Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3439 | 2024-08-22T02:24:07.431Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 47895
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:39092
req_id = 42420c84-2059-4982-bc34-0e37065c4625
response_code = 204
uri = /switch-ports
|
| 3440 | 2024-08-22T02:24:07.432Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 729
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:39092
req_id = ff5fe127-9915-4d83-b99d-ffe1345502a1
response_code = 200
uri = /network-bootstore-config
|
| 3441 | 2024-08-22T02:24:07.515Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:60056
|
| 3442 | 2024-08-22T02:24:07.515Z | INFO | SledAgent (ServiceManager): Setting up uplinkd service
file = sled-agent/src/services.rs:3956
|
| 3443 | 2024-08-22T02:24:07.540Z | INFO | SledAgent (ServiceManager): configuring port: HostPortConfig { port: "qsfp0", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }
file = sled-agent/src/services.rs:3967
|
| 3444 | 2024-08-22T02:24:07.562Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 47624
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:60056
req_id = 7abfeecb-f40f-4294-862e-9ea23267b666
response_code = 204
uri = /switch-ports
|
| 3445 | 2024-08-22T02:24:07.563Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 112
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:60056
req_id = d9efe1c3-93bd-4fbf-b8a1-ddc13a57063d
response_code = 200
uri = /network-bootstore-config
|
| 3446 | 2024-08-22T02:24:11.233Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:49286
|
| 3447 | 2024-08-22T02:24:11.234Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 534
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:49286
req_id = dbe8982c-dc57-478e-bd2a-6320dd871299
response_code = 200
uri = /instances/4b73965e-c980-4acb-a19e-b90a84025ee8/state
|
| 3448 | 2024-08-22T02:24:11.234Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-08-22T02:24:11.234575732Z }
file = sled-agent/src/instance.rs:600
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3449 | 2024-08-22T02:24:11.234Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:622
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
new_vmm_state = VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-08-22T02:24:11.234575732Z }
propolis_id = 20287db9-9bf9-4ca1-a2ef-9f74c529a27f
|
| 3450 | 2024-08-22T02:24:11.234Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:520
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
state = SledInstanceState { propolis_id: 20287db9-9bf9-4ca1-a2ef-9f74c529a27f (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-08-22T02:24:11.234575732Z }, migration_in: None, migration_out: None }
|
| 3451 | 2024-08-22T02:24:11.246Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-08-22T02:24:11.246016469Z }
file = sled-agent/src/instance.rs:600
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3452 | 2024-08-22T02:24:11.246Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:622
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
new_vmm_state = VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-08-22T02:24:11.246016469Z }
propolis_id = 20287db9-9bf9-4ca1-a2ef-9f74c529a27f
|
| 3453 | 2024-08-22T02:24:11.246Z | INFO | SledAgent (InstanceManager): terminating VMM that has exited
file = sled-agent/src/instance.rs:636
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3454 | 2024-08-22T02:24:11.246Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 3455 | 2024-08-22T02:24:11.246Z | WARN | SledAgent (kstat-sampler-worker): target removed with queued samples
file = oximeter/instruments/src/kstat/sampler.rs:557
id = 963434143452187269
n_samples = 6
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3456 | 2024-08-22T02:24:11.246Z | WARN | SledAgent (kstat-sampler-worker): target removed with queued samples
file = oximeter/instruments/src/kstat/sampler.rs:557
id = 14442580972254445587
n_samples = 6
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3457 | 2024-08-22T02:24:11.247Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:406
|
| 3458 | 2024-08-22T02:24:11.247Z | INFO | SledAgent (ZoneBundler): creating zone bundle
context = ZoneBundleContext { storage_dirs: ["/pool/int/6f31f2f8-a517-49b4-86c9-0d60b67db277/debug/bundle/zone", "/pool/int/97ca59dd-16d5-47a1-9104-ef732f7a459f/debug/bundle/zone"], cause: TerminatedInstance, extra_log_dirs: ["/pool/ext/a3847786-8a2e-45ee-87a5-075f35fc7eea/crypt/debug", "/pool/ext/c613840f-0c10-41fb-9afd-d136750700f5/crypt/debug", "/pool/ext/2beabcd5-0369-41ca-9d09-9a7ad5ca2181/crypt/debug", "/pool/ext/817bc2ac-fa69-4e9b-8fd6-6f77d7295e99/crypt/debug", "/pool/ext/50197822-d2db-4c18-aae1-d7d7bdd76af4/crypt/debug", "/pool/ext/39a6c1a0-fe6b-4779-af6d-7cf7ed3c68a2/crypt/debug", "/pool/ext/93a0a2e2-5fff-4e15-9016-1c60e061297f/crypt/debug", "/pool/ext/01770c46-d16d-4f08-bf5d-3bc7b219d504/crypt/debug", "/pool/ext/555e0f8f-2230-434a-8ce6-0ed459a335cf/crypt/debug"] }
file = sled-agent/src/zone_bundle.rs:339
zone_name = oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f
|
| 3459 | 2024-08-22T02:24:11.866Z | INFO | SledAgent (ZoneBundler): finished zone bundle
file = sled-agent/src/zone_bundle.rs:1137
metadata = ZoneBundleMetadata { id: ZoneBundleId { zone_name: "oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f", bundle_id: d73be5e8-f38e-4ea0-851b-5ff632bba173 }, time_created: 2024-08-22T02:24:11.247616936Z, version: 0, cause: TerminatedInstance }
|
| 3460 | 2024-08-22T02:24:11.866Z | WARN | SledAgent (InstanceManager): Halting and removing zone: oxz_propolis-server_20287db9-9bf9-4ca1-a2ef-9f74c529a27f
file = sled-agent/src/instance.rs:831
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3461 | 2024-08-22T02:24:12.979Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 7331
local_addr = [fd00:1122:3344:101::1]:42716
method = GET
remote_addr = [fd00:1122:3344:101::d]:54144
req_id = 4529692e-2530-4acb-8988-1bafd040b955
response_code = 200
sled_id = 37fb4e95-8c94-4aa8-946d-de6b4770caf9
uri = /37fb4e95-8c94-4aa8-946d-de6b4770caf9
|
| 3462 | 2024-08-22T02:24:13.629Z | INFO | SledAgent (InstanceManager): halt_and_remove_logged: Previous zone state: Running
file = illumos-utils/src/zone.rs:270
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3463 | 2024-08-22T02:24:13.647Z | INFO | SledAgent (PortManager): Removed route set for subnet
file = illumos-utils/src/opte/port_manager.rs:904
id = RouterId { vni: Vni(11127658), kind: System }
|
| 3464 | 2024-08-22T02:24:13.647Z | INFO | SledAgent (PortManager): Removed route set for subnet
file = illumos-utils/src/opte/port_manager.rs:904
id = RouterId { vni: Vni(11127658), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }
|
| 3465 | 2024-08-22T02:24:13.647Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:520
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
state = SledInstanceState { propolis_id: 20287db9-9bf9-4ca1-a2ef-9f74c529a27f (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-08-22T02:24:13.647653314Z }, migration_in: None, migration_out: None }
|
| 3466 | 2024-08-22T02:24:13.667Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:520
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
state = SledInstanceState { propolis_id: 20287db9-9bf9-4ca1-a2ef-9f74c529a27f (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-08-22T02:24:13.647653314Z }, migration_in: None, migration_out: None }
|
| 3467 | 2024-08-22T02:24:13.671Z | INFO | SledAgent (InstanceManager): State monitoring task complete
file = sled-agent/src/instance.rs:768
instance_id = 4b73965e-c980-4acb-a19e-b90a84025ee8
|
| 3468 | 2024-08-22T02:24:14.074Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:58734
|
| 3469 | 2024-08-22T02:24:14.074Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 83
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:58734
req_id = 484444ac-f102-425d-9be6-c245f8c0ad42
response_code = 200
uri = /vpc-routes
|
| 3470 | 2024-08-22T02:24:14.081Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 696
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:58734
req_id = 4d972a1a-cdca-4b80-a816-e9029bd65ea8
response_code = 204
uri = /vpc-routes
|
| 3471 | 2024-08-22T02:24:14.091Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:64449
|
| 3472 | 2024-08-22T02:24:14.091Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 101
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:64449
req_id = e5041b53-5c93-4c46-8a07-e87e1fb7f2e5
response_code = 200
uri = /v2p
|
| 3473 | 2024-08-22T02:24:15.406Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:48720
|
| 3474 | 2024-08-22T02:24:15.406Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 173
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:48720
req_id = 09d27d33-cffd-42b1-a2dc-f977c53dffe4
response_code = 200
uri = /v2p
|
| 3475 | 2024-08-22T02:24:16.208Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:62457
|
| 3476 | 2024-08-22T02:24:16.208Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 91
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:62457
req_id = b95a315e-4527-4610-8bde-f2d3f4b481b7
response_code = 200
uri = /vpc-routes
|
| 3477 | 2024-08-22T02:24:16.215Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914
latency_us = 770
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:62457
req_id = 8aa7be98-ba66-4eaf-8a08-8181ef8f8558
response_code = 204
uri = /vpc-routes
|