| 1 | | [ Apr 3 22:19:12 Enabled. ] |
| 2 | | [ Apr 3 22:19:12 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] |
| 3 | | [ Apr 3 22:19:12 Method "start" exited with status 0. ] |
| 4 | | [ Apr 3 22:19:12 Rereading configuration. ] |
| 5 | | [ Apr 3 22:19:12 No 'refresh' method defined. Treating as :true. ] |
| 6 | | note: configured to log to "/dev/stdout" |
| 7 | 2024-04-03T22:19:12.294Z | INFO | SledAgent: Ensuring contract reaper thread
file = illumos-utils/src/running_zone.rs:163
|
| 8 | 2024-04-03T22:19:12.294Z | INFO | SledAgent (BootstrapAgentStartup): finding links ["net0", "net1"]
file = sled-agent/src/bootstrap/pre_server.rs:253
|
| 9 | 2024-04-03T22:19:12.347Z | INFO | SledAgent (BootstrapAgentStartup): Starting mg-ddm service
addr-objs = [AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]
file = sled-agent/src/bootstrap/pre_server.rs:260
|
| 10 | 2024-04-03T22:19:12.347Z | INFO | SledAgent (BootstrapAgentStartup): Importing mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:42
path = /opt/oxide/mg-ddm/pkg/ddm/manifest.xml
|
| 11 | 2024-04-03T22:19:12.463Z | INFO | SledAgent (BootstrapAgentStartup): Setting mg-ddm interfaces
file = sled-agent/src/bootstrap/maghemite.rs:50
interfaces = ("net0/ll" "net1/ll")
|
| 12 | 2024-04-03T22:19:12.472Z | INFO | SledAgent (BootstrapAgentStartup): Enabling mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:56
|
| 13 | 2024-04-03T22:19:12.480Z | INFO | SledAgent (BootstrapAgentStartup): Ensuring zfs key directory exists
file = sled-agent/src/bootstrap/pre_server.rs:270
path = /var/run/oxide/
|
| 14 | 2024-04-03T22:19:13.082Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }
|
| 15 | 2024-04-03T22:19:13.220Z | INFO | SledAgent: Starting KeyManager
file = sled-agent/src/long_running_tasks.rs:115
|
| 16 | 2024-04-03T22:19:13.220Z | INFO | SledAgent: Starting StorageManager
file = sled-agent/src/long_running_tasks.rs:127
|
| 17 | 2024-04-03T22:19:13.220Z | INFO | SledAgent: Starting StorageMonitor
file = sled-agent/src/long_running_tasks.rs:137
|
| 18 | 2024-04-03T22:19:13.220Z | INFO | SledAgent: Starting HardwareManager
file = sled-agent/src/long_running_tasks.rs:157
sled_mode = Scrimlet { asic: SoftNpuZone }
|
| 19 | 2024-04-03T22:19:13.220Z | INFO | SledAgent (DumpSetup-archival): DumpSetup poll loop started.
file = sled-agent/src/dump_setup.rs:291
|
| 20 | 2024-04-03T22:19:13.220Z | INFO | SledAgent (HardwareManager): Creating HardwareManager
file = sled-hardware/src/illumos/mod.rs:609
|
| 21 | 2024-04-03T22:19:13.220Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:801
|
| 22 | 2024-04-03T22:19:13.223Z | INFO | SledAgent (HardwareManager): Generated i86pc baseboard Pc { identifier: "buskin", model: "i86pc" }
file = sled-hardware/src/illumos/mod.rs:535
|
| 23 | 2024-04-03T22:19:13.224Z | WARN | SledAgent (HardwareManager): Device is not a Gimlet (i86pc), proceeding with null hardware view
file = sled-hardware/src/illumos/mod.rs:658
|
| 24 | 2024-04-03T22:19:13.224Z | INFO | SledAgent: Starting HardwareMonitor
file = sled-agent/src/long_running_tasks.rs:171
|
| 25 | 2024-04-03T22:19:13.224Z | INFO | SledAgent (HardwareManager): Monitoring for hardware updates
file = sled-hardware/src/illumos/mod.rs:716
|
| 26 | 2024-04-03T22:19:13.224Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:227
vdev = /scratch/m2_0.vdev
|
| 27 | 2024-04-03T22:19:13.224Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
file = sled-agent/src/hardware_monitor.rs:261
underlay_network_info = None
|
| 28 | 2024-04-03T22:19:13.224Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 29 | 2024-04-03T22:19:13.224Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }
|
| 30 | 2024-04-03T22:19:13.224Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:111
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }
path = /scratch/m2_0.vdev
|
| 31 | 2024-04-03T22:19:13.236Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev
file = sled-hardware/src/disk.rs:301
|
| 32 | 2024-04-03T22:19:13.236Z | INFO | SledAgent (StorageResources): Formatting zpool with generated ID
file = sled-hardware/src/disk.rs:312
id = 38c50e2b-3905-4ab7-a650-3ad6b1287295
|
| 33 | 2024-04-03T22:19:13.337Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal }
|
| 34 | 2024-04-03T22:19:13.536Z | INFO | SledAgent (StorageManager): Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }
file = sled-storage/src/manager.rs:381
|
| 35 | 2024-04-03T22:19:13.536Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:227
vdev = /scratch/m2_1.vdev
|
| 36 | 2024-04-03T22:19:13.536Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }
|
| 37 | 2024-04-03T22:19:13.536Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 38 | 2024-04-03T22:19:13.536Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }
|
| 39 | 2024-04-03T22:19:13.536Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:111
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }
path = /scratch/m2_1.vdev
|
| 40 | 2024-04-03T22:19:13.544Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev
file = sled-hardware/src/disk.rs:301
|
| 41 | 2024-04-03T22:19:13.544Z | INFO | SledAgent (StorageResources): Formatting zpool with generated ID
file = sled-hardware/src/disk.rs:312
id = 939cb7b8-5f2f-4f5c-821a-6bf92b625674
|
| 42 | 2024-04-03T22:19:13.629Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal }
|
| 43 | 2024-04-03T22:19:13.827Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:227
vdev = /scratch/u2_0.vdev
|
| 44 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 45 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 46 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
|
| 47 | 2024-04-03T22:19:13.827Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:227
vdev = /scratch/u2_1.vdev
|
| 48 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 49 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 50 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
|
| 51 | 2024-04-03T22:19:13.827Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:227
vdev = /scratch/u2_2.vdev
|
| 52 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 53 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 54 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
|
| 55 | 2024-04-03T22:19:13.827Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:227
vdev = /scratch/u2_3.vdev
|
| 56 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 57 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 58 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
|
| 59 | 2024-04-03T22:19:13.827Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:227
vdev = /scratch/u2_4.vdev
|
| 60 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, variant: U2, slot: 1030 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 61 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
|
| 62 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 63 | 2024-04-03T22:19:13.827Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:227
vdev = /scratch/u2_5.vdev
|
| 64 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, variant: U2, slot: 1031 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 65 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
|
| 66 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 67 | 2024-04-03T22:19:13.827Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:227
vdev = /scratch/u2_6.vdev
|
| 68 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }, variant: U2, slot: 1032 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 69 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
|
| 70 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 71 | 2024-04-03T22:19:13.827Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:227
vdev = /scratch/u2_7.vdev
|
| 72 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }, variant: U2, slot: 1033 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 73 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
|
| 74 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 75 | 2024-04-03T22:19:13.827Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:227
vdev = /scratch/u2_8.vdev
|
| 76 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }, variant: U2, slot: 1034 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 77 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 78 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
|
| 79 | 2024-04-03T22:19:13.827Z | INFO | SledAgent: Waiting for boot disk
file = sled-agent/src/long_running_tasks.rs:87
|
| 80 | 2024-04-03T22:19:13.827Z | INFO | SledAgent: Found boot disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }
file = sled-agent/src/long_running_tasks.rs:89
|
| 81 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 82 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 83 | 2024-04-03T22:19:13.827Z | INFO | SledAgent: Starting Bootstore
file = sled-agent/src/long_running_tasks.rs:195
|
| 84 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (bootstore): No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:188
peer_id = pc-buskin-i86pc
|
| 85 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (bootstore): No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:188
peer_id = pc-buskin-i86pc
|
| 86 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (bootstore): No ledger found. Loading Fsm::State as Uninitialized
file = bootstore/src/schemes/v0/storage.rs:83
peer_id = pc-buskin-i86pc
|
| 87 | 2024-04-03T22:19:13.827Z | INFO | SledAgent (bootstore): No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/cluster/bootstore-network-config.json
file = common/src/ledger.rs:188
peer_id = pc-buskin-i86pc
|
| 88 | 2024-04-03T22:19:13.828Z | INFO | SledAgent (bootstore): No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/cluster/bootstore-network-config.json
file = common/src/ledger.rs:188
peer_id = pc-buskin-i86pc
|
| 89 | 2024-04-03T22:19:13.828Z | INFO | SledAgent (bootstore): No ledger found for network config
file = bootstore/src/schemes/v0/storage.rs:146
peer_id = pc-buskin-i86pc
|
| 90 | 2024-04-03T22:19:13.828Z | INFO | SledAgent: Starting Bootstore DDMD poller
file = sled-agent/src/long_running_tasks.rs:200
|
| 91 | 2024-04-03T22:19:13.828Z | INFO | SledAgent: Starting ZoneBundler related tasks
file = sled-agent/src/long_running_tasks.rs:215
|
| 92 | 2024-04-03T22:19:13.837Z | INFO | SledAgent (ServiceManager): Creating ServiceManager
file = sled-agent/src/services.rs:709
|
| 93 | 2024-04-03T22:19:13.838Z | INFO | SledAgent (auto-cleanup-task): top of bundle cleanup loop
file = sled-agent/src/zone_bundle.rs:289
next_cleanup = Instant { tv_sec: 1615, tv_nsec: 896857368 }
time_to_next_cleanup = 599.999931608s
|
| 94 | 2024-04-03T22:19:13.838Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3596
|
| 95 | 2024-04-03T22:19:13.838Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 96 | 2024-04-03T22:19:13.838Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/sled-agent-request.json
file = common/src/ledger.rs:188
|
| 97 | 2024-04-03T22:19:13.838Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/sled-agent-request.json
file = common/src/ledger.rs:188
|
| 98 | 2024-04-03T22:19:13.840Z | INFO | SledAgent (dropshot (BootstrapAgent)): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
|
| 99 | 2024-04-03T22:19:13.840Z | INFO | SledAgent (SledAgentSprocketsServer): Started listening
file = sled-agent/src/bootstrap/sprockets_server.rs:44
local_addr = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 100 | 2024-04-03T22:19:13.840Z | INFO | SledAgent (RSS): Injecting RSS configuration: RackInitializeRequest {
trust_quorum_peers: None,
bootstrap_discovery: OnlyOurs,
ntp_servers: [
"0.pool.ntp.org",
],
dns_servers: [
1.1.1.1,
9.9.9.9,
],
internal_services_ip_pool_ranges: [
V4(
Ipv4Range {
first: 10.151.1.164,
last: 10.151.1.173,
},
),
],
external_dns_ips: [
10.151.1.164,
10.151.1.165,
],
external_dns_zone_name: "oxide.test",
external_certificates: "<redacted>",
recovery_silo: RecoverySiloConfig {
silo_name: Name(
"recovery",
),
user_name: UserId(
"recovery",
),
user_password_hash: NewPasswordHash(
PasswordHashString {
string: "$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY",
encoding: B64,
},
),
},
rack_network_config: RackNetworkConfigV1 {
rack_subnet: Ipv6Network {
addr: fd00:1122:3344:100::,
prefix: 56,
},
infra_ip_first: 10.151.1.174,
infra_ip_last: 10.151.1.174,
ports: [
PortConfigV1 {
routes: [
RouteConfig {
destination: V4(
Ipv4Network {
addr: 0.0.0.0,
prefix: 0,
},
),
nexthop: 10.151.1.100,
},
],
addresses: [
V4(
Ipv4Network {
addr: 10.151.1.174,
prefix: 24,
},
),
],
switch: Switch0,
port: "qsfp0",
uplink_port_speed: Speed40G,
uplink_port_fec: None,
bgp_peers: [],
autoneg: false,
},
],
bgp: [],
bfd: [],
},
}
file = sled-agent/src/rack_setup/service.rs:997
|
| 101 | 2024-04-03T22:19:13.840Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
| 102 | 2024-04-03T22:19:13.840Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 103 | 2024-04-03T22:19:13.840Z | INFO | SledAgent (RSS): No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/rss-plan-completed.marker
file = common/src/ledger.rs:188
|
| 104 | 2024-04-03T22:19:13.840Z | INFO | SledAgent (RSS): No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/rss-plan-completed.marker
file = common/src/ledger.rs:188
|
| 105 | 2024-04-03T22:19:13.840Z | INFO | SledAgent (RSS): RSS configuration has not been fully applied yet
file = sled-agent/src/rack_setup/service.rs:1057
|
| 106 | 2024-04-03T22:19:13.840Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 107 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (RSS): No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/rss-sled-plan.json
file = common/src/ledger.rs:188
|
| 108 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (RSS): No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/rss-sled-plan.json
file = common/src/ledger.rs:188
|
| 109 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (RSS): Creating new allocation plan
file = sled-agent/src/rack_setup/service.rs:1098
|
| 110 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (RSS): Serializing plan
file = sled-agent/src/rack_setup/plan/sled.rs:118
|
| 111 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (RSS): Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1
file = sled-agent/src/rack_setup/plan/sled.rs:91
|
| 112 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 113 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.rss-sled-plan.json.tmp
file = common/src/ledger.rs:199
|
| 114 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.rss-sled-plan.json.tmp
file = common/src/ledger.rs:199
|
| 115 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (RSS): Sled plan written to storage
file = sled-agent/src/rack_setup/plan/sled.rs:138
|
| 116 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (RSS): Writing Rack Network Configuration to bootstore
file = sled-agent/src/rack_setup/service.rs:1131
|
| 117 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 1, current_generation: 0
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-buskin-i86pc
|
| 118 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-buskin-i86pc
|
| 119 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-buskin-i86pc
|
| 120 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 1
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-buskin-i86pc
|
| 121 | 2024-04-03T22:19:13.841Z | INFO | SledAgent (BootstrapAgentRssHandler): Received initialization request from RSS
file = sled-agent/src/bootstrap/rss_handle.rs:217
request = StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 1ee9e643-97de-413d-8b33-615f41c711a3, rack_id: 6418b01e-3d3b-4688-85d2-17d265929a3a, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }
target_sled = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 122 | 2024-04-03T22:19:13.842Z | INFO | SledAgent (SledAgentSprocketsServer): Accepted connection
file = sled-agent/src/bootstrap/sprockets_server.rs:66
remote_addr = [fdb0:18c0:4d0d:9fb2::1]:33406
|
| 123 | 2024-04-03T22:19:13.842Z | INFO | SledAgent (SledAgentMain): Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 1ee9e643-97de-413d-8b33-615f41c711a3, rack_id: 6418b01e-3d3b-4688-85d2-17d265929a3a, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }
file = sled-agent/src/bootstrap/server.rs:356
|
| 124 | 2024-04-03T22:19:13.842Z | INFO | SledAgent (SledAgentMain): KeyManager: using hardcoded secret retriever
file = sled-agent/src/bootstrap/server.rs:371
|
| 125 | 2024-04-03T22:19:13.842Z | INFO | SledAgent (StorageManager): Received KeyManagerReady
file = sled-storage/src/manager.rs:381
|
| 126 | 2024-04-03T22:19:13.842Z | INFO | SledAgent: setting up sled agent server
file = sled-agent/src/server.rs:44
|
| 127 | 2024-04-03T22:19:13.842Z | INFO | SledAgent (StorageManager): Loading storage config
file = sled-storage/src/manager.rs:530
|
| 128 | 2024-04-03T22:19:13.842Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
| 129 | 2024-04-03T22:19:13.842Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }
|
| 130 | 2024-04-03T22:19:13.842Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = load_ledger
|
| 131 | 2024-04-03T22:19:13.842Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = load_ledger
|
| 132 | 2024-04-03T22:19:13.842Z | INFO | SledAgent (StorageManager): No ledger of physical disks exists
file = sled-storage/src/manager.rs:519
|
| 133 | 2024-04-03T22:19:13.842Z | INFO | SledAgent (StorageManager): KeyManager ready, but no ledger detected
file = sled-storage/src/manager.rs:547
|
| 134 | 2024-04-03T22:19:13.845Z | INFO | SledAgent (ServiceManager): Enabling switch zone (new)
file = sled-agent/src/services.rs:3827
|
| 135 | 2024-04-03T22:19:13.853Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
|
| 136 | 2024-04-03T22:19:13.862Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
|
| 137 | 2024-04-03T22:19:13.873Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
|
| 138 | 2024-04-03T22:19:13.883Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
|
| 139 | 2024-04-03T22:19:13.891Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
|
| 140 | 2024-04-03T22:19:13.900Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
|
| 141 | 2024-04-03T22:19:13.909Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
|
| 142 | 2024-04-03T22:19:13.919Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
|
| 143 | 2024-04-03T22:19:13.927Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
|
| 144 | 2024-04-03T22:19:13.927Z | INFO | SledAgent (StorageManager): No disks to be automatically managed
file = sled-storage/src/manager.rs:558
|
| 145 | 2024-04-03T22:19:13.927Z | INFO | SledAgent (StorageManager): automatically managing disks
file = sled-storage/src/manager.rs:370
|
| 146 | 2024-04-03T22:19:13.927Z | INFO | SledAgent (StorageResources): Synchronizing disk managment
file = sled-storage/src/resources.rs:339
|
| 147 | 2024-04-03T22:19:13.927Z | INFO | SledAgent (StorageManager): Successfully synchronized disks without error
file = sled-storage/src/manager.rs:449
result = DisksManagementResult { status: [] }
|
| 148 | 2024-04-03T22:19:13.927Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 149 | 2024-04-03T22:19:13.948Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:315
|
| 150 | 2024-04-03T22:19:13.966Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:354
|
| 151 | 2024-04-03T22:19:13.985Z | INFO | SledAgent: SledAgent::new(..) starting
file = sled-agent/src/sled_agent.rs:355
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 152 | 2024-04-03T22:19:13.985Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 153 | 2024-04-03T22:19:13.985Z | INFO | SledAgent: Requested swap device of size 64 GiB
file = sled-agent/src/sled_agent.rs:367
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 154 | 2024-04-03T22:19:13.985Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 155 | 2024-04-03T22:19:13.988Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 74.82811ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "74835456-7ded-40b3-a5f8-e4c640ec0143", "content-length": "124", "date": "Wed, 03 Apr 2024 22:19:13 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "74835456-7ded-40b3-a5f8-e4c640ec0143" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 156 | 2024-04-03T22:19:14.000Z | INFO | SledAgent: attempting to create encrypted zvol: name="oxi_38c50e2b-3905-4ab7-a650-3ad6b1287295/swap", size_gb=64
file = sled-agent/src/swap_device.rs:153
|
| 157 | 2024-04-03T22:19:14.024Z | INFO | SledAgent: successfully created encrypted zvol: name="oxi_38c50e2b-3905-4ab7-a650-3ad6b1287295/swap", size_gb=64
file = sled-agent/src/swap_device.rs:249
|
| 158 | 2024-04-03T22:19:14.024Z | INFO | SledAgent: adding swap device: swapname="/dev/zvol/dsk/oxi_38c50e2b-3905-4ab7-a650-3ad6b1287295/swap"
file = sled-agent/src/swap_device.rs:96
|
| 159 | 2024-04-03T22:19:14.029Z | INFO | SledAgent: Mounting backing filesystems
file = sled-agent/src/sled_agent.rs:382
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 160 | 2024-04-03T22:19:14.029Z | INFO | SledAgent (BackingFs): Processing fmd
file = sled-agent/src/backing_fs.rs:126
|
| 161 | 2024-04-03T22:19:14.029Z | INFO | SledAgent (BackingFs): Ensuring dataset oxi_38c50e2b-3905-4ab7-a650-3ad6b1287295/backing/fmd
file = sled-agent/src/backing_fs.rs:136
|
| 162 | 2024-04-03T22:19:14.064Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 163 | 2024-04-03T22:19:14.064Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 78.68691ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d3a2da66-2064-46e5-9bd0-d65b98c55bb2", "content-length": "124", "date": "Wed, 03 Apr 2024 22:19:13 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d3a2da66-2064-46e5-9bd0-d65b98c55bb2" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 164 | 2024-04-03T22:19:14.094Z | INFO | SledAgent (BackingFs): Stopping service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:170
|
| 165 | 2024-04-03T22:19:14.144Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 166 | 2024-04-03T22:19:14.144Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 201.645148ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e048e8e5-5932-4ac5-8ff7-8e4d1b00b881", "content-length": "124", "date": "Wed, 03 Apr 2024 22:19:14 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e048e8e5-5932-4ac5-8ff7-8e4d1b00b881" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 167 | 2024-04-03T22:19:14.311Z | INFO | SledAgent (BackingFs): Mounting oxi_38c50e2b-3905-4ab7-a650-3ad6b1287295/backing/fmd on /var/fm/fmd
file = sled-agent/src/backing_fs.rs:178
|
| 168 | 2024-04-03T22:19:14.323Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/rsrc
file = sled-agent/src/backing_fs.rs:186
|
| 169 | 2024-04-03T22:19:14.323Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/ckpt
file = sled-agent/src/backing_fs.rs:186
|
| 170 | 2024-04-03T22:19:14.323Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/xprt
file = sled-agent/src/backing_fs.rs:186
|
| 171 | 2024-04-03T22:19:14.323Z | INFO | SledAgent (BackingFs): Starting service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:192
|
| 172 | 2024-04-03T22:19:14.347Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 173 | 2024-04-03T22:19:14.347Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 145.499706ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "8d7740a9-d9b2-4fed-a329-f158d2526a39", "content-length": "124", "date": "Wed, 03 Apr 2024 22:19:14 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "8d7740a9-d9b2-4fed-a329-f158d2526a39" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 174 | 2024-04-03T22:19:14.494Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 175 | 2024-04-03T22:19:14.495Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 147.396109ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "9d5d1cdb-af7a-4f90-8e79-566ccded8cf9", "content-length": "124", "date": "Wed, 03 Apr 2024 22:19:14 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "9d5d1cdb-af7a-4f90-8e79-566ccded8cf9" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 176 | 2024-04-03T22:19:14.643Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 177 | 2024-04-03T22:19:14.643Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 239.213796ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "9724f807-e8ca-4bf6-98da-193dcdb9a08a", "content-length": "124", "date": "Wed, 03 Apr 2024 22:19:14 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "9724f807-e8ca-4bf6-98da-193dcdb9a08a" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 178 | 2024-04-03T22:19:14.884Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 179 | 2024-04-03T22:19:14.885Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 156.051536ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "1a1f9eea-823f-40c6-9014-89ab04633ccf", "content-length": "124", "date": "Wed, 03 Apr 2024 22:19:14 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "1a1f9eea-823f-40c6-9014-89ab04633ccf" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 180 | 2024-04-03T22:19:15.026Z | INFO | SledAgent: using '[AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]' as data links for xde driver
file = illumos-utils/src/opte/illumos.rs:90
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 181 | 2024-04-03T22:19:15.042Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 182 | 2024-04-03T22:19:15.077Z | INFO | SledAgent (VmmReservoirManager): 50% of 102952026112 physical ram = 51476013056 bytes)
file = sled-agent/src/vmm_reservoir.rs:266
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 183 | 2024-04-03T22:19:15.077Z | INFO | SledAgent (VmmReservoirManager): Setting reservoir size to 49090 MiB bytes
file = sled-agent/src/vmm_reservoir.rs:274
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 184 | 2024-04-03T22:19:15.077Z | INFO | SledAgent (ServiceManager): sled agent started
file = sled-agent/src/services.rs:991
underlay_address = fd00:1122:3344:101::1
|
| 185 | 2024-04-03T22:19:15.077Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
| 186 | 2024-04-03T22:19:15.078Z | INFO | SledAgent (dropshot (SledAgent)): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195
local_addr = [fd00:1122:3344:101::1]:12345
|
| 187 | 2024-04-03T22:19:15.078Z | INFO | SledAgent (SledAgentMain): Sled Agent loaded; recording configuration
file = sled-agent/src/bootstrap/server.rs:426
|
| 188 | 2024-04-03T22:19:15.078Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 189 | 2024-04-03T22:19:15.078Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:199
|
| 190 | 2024-04-03T22:19:15.078Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:199
|
| 191 | 2024-04-03T22:19:15.079Z | INFO | SledAgent (HardwareMonitor): Sled Agent Started
file = sled-agent/src/hardware_monitor.rs:138
|
| 192 | 2024-04-03T22:19:15.079Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 193 | 2024-04-03T22:19:15.079Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
file = sled-agent/src/hardware_monitor.rs:261
underlay_network_info = Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.1.100 }], addresses: [V4(Ipv4Network { addr: 10.151.1.174, prefix: 24 })], switch: Switch0, port: "qsfp0", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))
|
| 194 | 2024-04-03T22:19:15.079Z | WARN | SledAgent: failed to register as a metric producer with Nexus
error = "Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/sled_agent.rs:1190
retry_after = 230.652165ms
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 195 | 2024-04-03T22:19:15.079Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3596
|
| 196 | 2024-04-03T22:19:15.079Z | INFO | SledAgent (SledAgentSprocketsServer): Connection closed
file = sled-agent/src/bootstrap/sprockets_server.rs:73
remote_addr = [fdb0:18c0:4d0d:9fb2::1]:33406
|
| 197 | 2024-04-03T22:19:15.285Z | INFO | SledAgent (BootstrapAgentRssHandler): Peer agent initialized
file = sled-agent/src/bootstrap/rss_handle.rs:108
peer = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 198 | 2024-04-03T22:19:15.285Z | INFO | SledAgent (BootstrapAgentRssHandler): Initialized sled agent
file = sled-agent/src/bootstrap/rss_handle.rs:232
target_sled = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 199 | 2024-04-03T22:19:15.285Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 200 | 2024-04-03T22:19:15.285Z | INFO | SledAgent (RSS): No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/rss-service-plan-v3.json
file = common/src/ledger.rs:188
|
| 201 | 2024-04-03T22:19:15.285Z | INFO | SledAgent (RSS): No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/rss-service-plan-v3.json
file = common/src/ledger.rs:188
|
| 202 | 2024-04-03T22:19:15.285Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 203 | 2024-04-03T22:19:15.285Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 204 | 2024-04-03T22:19:15.311Z | WARN | SledAgent: failed to register as a metric producer with Nexus
error = "Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/sled_agent.rs:1190
retry_after = 685.625746ms
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 205 | 2024-04-03T22:19:15.317Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:47889
|
| 206 | 2024-04-03T22:19:15.317Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 207 | 2024-04-03T22:19:15.317Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 138
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:47889
req_id = 6df91ebc-c2cb-46a3-92a5-6f446320a371
response_code = 200
uri = /inventory
|
| 208 | 2024-04-03T22:19:15.354Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:57290
|
| 209 | 2024-04-03T22:19:15.354Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 31
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:57290
req_id = 6b9b0982-9bf6-43cf-8c7e-abf6b4f6d409
response_code = 200
uri = /sled-role
|
| 210 | 2024-04-03T22:19:15.355Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 211 | 2024-04-03T22:19:15.355Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.rss-service-plan-v3.json.tmp
file = common/src/ledger.rs:199
|
| 212 | 2024-04-03T22:19:15.355Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.rss-service-plan-v3.json.tmp
file = common/src/ledger.rs:199
|
| 213 | 2024-04-03T22:19:15.356Z | INFO | SledAgent (RSS): Service plan written to storage
file = sled-agent/src/rack_setup/plan/service.rs:802
|
| 214 | 2024-04-03T22:19:15.386Z | INFO | SledAgent (RSS): attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 60971685-a36c-4f5c-9601-4f24fe9e6b1a, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, pool_id: 7d954909-2e42-44b1-96d9-f536366f6758 }, OmicronPhysicalDiskConfig { id: 8ceab58c-1296-4cee-99a9-3edc6cda9420, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, pool_id: 0c6b0894-9c77-422a-8363-2705812a672c }, OmicronPhysicalDiskConfig { id: 6f9bdef3-12d3-45ca-a1cb-e7d1c3518880, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, pool_id: af9b6346-5d28-46df-baea-640dad9b76c3 }, OmicronPhysicalDiskConfig { id: 634b2092-89fb-47fb-8789-2fa53e55f4bb, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, pool_id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e }, OmicronPhysicalDiskConfig { id: 024dc529-273d-481e-a961-c69d2315d5d9, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, pool_id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a }, OmicronPhysicalDiskConfig { id: bb568e2f-278a-44b1-9f4e-4fb26a5c9265, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, pool_id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4 }, OmicronPhysicalDiskConfig { id: 69dc89c3-a111-4e05-9cb2-7a24d8f4c29d, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }, pool_id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8 }, OmicronPhysicalDiskConfig { id: baa01d53-698b-4890-8779-0595a5dcd58a, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }, pool_id: 70a16060-4587-476f-9772-16ea3a5a39aa }, OmicronPhysicalDiskConfig { id: c07aaaa5-d119-45c6-a06e-5772e5c4b5cf, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }, pool_id: 761d3524-c333-4332-9563-7d1ceb36709e }], generation: Generation(1) }
file = sled-agent/src/rack_setup/service.rs:344
sled_address = [fd00:1122:3344:101::1]:12345
|
| 215 | 2024-04-03T22:19:15.386Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:46916
|
| 216 | 2024-04-03T22:19:15.387Z | INFO | SledAgent (StorageManager): Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, id: 60971685-a36c-4f5c-9601-4f24fe9e6b1a, pool_id: 7d954909-2e42-44b1-96d9-f536366f6758 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, id: 8ceab58c-1296-4cee-99a9-3edc6cda9420, pool_id: 0c6b0894-9c77-422a-8363-2705812a672c }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, id: 6f9bdef3-12d3-45ca-a1cb-e7d1c3518880, pool_id: af9b6346-5d28-46df-baea-640dad9b76c3 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, id: 634b2092-89fb-47fb-8789-2fa53e55f4bb, pool_id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, id: 024dc529-273d-481e-a961-c69d2315d5d9, pool_id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, id: bb568e2f-278a-44b1-9f4e-4fb26a5c9265, pool_id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }, id: 69dc89c3-a111-4e05-9cb2-7a24d8f4c29d, pool_id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }, id: baa01d53-698b-4890-8779-0595a5dcd58a, pool_id: 70a16060-4587-476f-9772-16ea3a5a39aa }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }, id: c07aaaa5-d119-45c6-a06e-5772e5c4b5cf, pool_id: 761d3524-c333-4332-9563-7d1ceb36709e }] }, tx: ... }
file = sled-storage/src/manager.rs:381
|
| 217 | 2024-04-03T22:19:15.387Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = omicron_physical_disks_ensure
|
| 218 | 2024-04-03T22:19:15.387Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = omicron_physical_disks_ensure
|
| 219 | 2024-04-03T22:19:15.387Z | INFO | SledAgent (StorageManager): No previously-stored 'requested disks', creating new ledger
file = sled-storage/src/manager.rs:681
request = omicron_physical_disks_ensure
|
| 220 | 2024-04-03T22:19:15.387Z | INFO | SledAgent (StorageResources): Synchronizing disk managment
file = sled-storage/src/resources.rs:339
|
| 221 | 2024-04-03T22:19:15.387Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
| 222 | 2024-04-03T22:19:15.387Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 223 | 2024-04-03T22:19:15.387Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:111
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
path = /scratch/u2_0.vdev
|
| 224 | 2024-04-03T22:19:15.496Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev
file = sled-hardware/src/disk.rs:301
|
| 225 | 2024-04-03T22:19:15.496Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:307
id = 7d954909-2e42-44b1-96d9-f536366f6758
|
| 226 | 2024-04-03T22:19:16.178Z | WARN | SledAgent: failed to register as a metric producer with Nexus
error = "Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/sled_agent.rs:1190
retry_after = 1.238479651s
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 227 | 2024-04-03T22:19:16.178Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 228 | 2024-04-03T22:19:16.625Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }
|
| 229 | 2024-04-03T22:19:16.822Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
| 230 | 2024-04-03T22:19:16.822Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 231 | 2024-04-03T22:19:16.822Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 232 | 2024-04-03T22:19:16.822Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 233 | 2024-04-03T22:19:16.822Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 234 | 2024-04-03T22:19:16.822Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:38
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 235 | 2024-04-03T22:19:16.822Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 236 | 2024-04-03T22:19:17.296Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 237 | 2024-04-03T22:19:17.296Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 238 | 2024-04-03T22:19:17.506Z | WARN | SledAgent: failed to register as a metric producer with Nexus
error = "Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/sled_agent.rs:1190
retry_after = 1.419221256s
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 239 | 2024-04-03T22:19:17.509Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:57
|
| 240 | 2024-04-03T22:19:17.736Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_7d954909-2e42-44b1-96d9-f536366f6758/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 241 | 2024-04-03T22:19:17.919Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_switch
|
| 242 | 2024-04-03T22:19:18.559Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 243 | 2024-04-03T22:19:18.767Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 55.399324ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 244 | 2024-04-03T22:19:19.164Z | WARN | SledAgent: failed to register as a metric producer with Nexus
error = "Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/sled_agent.rs:1190
retry_after = 4.323909911s
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 245 | 2024-04-03T22:19:19.164Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 246 | 2024-04-03T22:19:19.165Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 60.364809ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 247 | 2024-04-03T22:19:19.865Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 248 | 2024-04-03T22:19:19.865Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 233.709658ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 249 | 2024-04-03T22:19:20.684Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 201.432853ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 250 | 2024-04-03T22:19:21.107Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_7d954909-2e42-44b1-96d9-f536366f6758
file = sled-storage/src/dataset.rs:480
|
| 251 | 2024-04-03T22:19:21.107Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 252 | 2024-04-03T22:19:21.319Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 799.078609ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 253 | 2024-04-03T22:19:21.322Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 254 | 2024-04-03T22:19:21.327Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
| 255 | 2024-04-03T22:19:21.327Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
| 256 | 2024-04-03T22:19:21.327Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
| 257 | 2024-04-03T22:19:21.327Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 258 | 2024-04-03T22:19:21.327Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:111
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
path = /scratch/u2_1.vdev
|
| 259 | 2024-04-03T22:19:21.526Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev
file = sled-hardware/src/disk.rs:301
|
| 260 | 2024-04-03T22:19:21.526Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:307
id = 0c6b0894-9c77-422a-8363-2705812a672c
|
| 261 | 2024-04-03T22:19:22.621Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }
|
| 262 | 2024-04-03T22:19:22.834Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
| 263 | 2024-04-03T22:19:22.835Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 264 | 2024-04-03T22:19:22.835Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 265 | 2024-04-03T22:19:22.835Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 266 | 2024-04-03T22:19:22.835Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 267 | 2024-04-03T22:19:22.835Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:38
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 268 | 2024-04-03T22:19:22.835Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 269 | 2024-04-03T22:19:23.348Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 270 | 2024-04-03T22:19:23.349Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 271 | 2024-04-03T22:19:23.805Z | WARN | SledAgent: failed to register as a metric producer with Nexus
error = "Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/sled_agent.rs:1190
retry_after = 10.163736533s
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 272 | 2024-04-03T22:19:24.295Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:57
|
| 273 | 2024-04-03T22:19:24.763Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 274 | 2024-04-03T22:19:24.763Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_0c6b0894-9c77-422a-8363-2705812a672c/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 275 | 2024-04-03T22:19:25.413Z | INFO | SledAgent (ServiceManager): Ensuring tfportrear0_0/ll exists in zone
file = sled-agent/src/services.rs:2294
|
| 276 | 2024-04-03T22:19:25.617Z | INFO | SledAgent (VmmReservoirManager): Finished setting reservoir size to 49090 MiB bytes
file = sled-agent/src/vmm_reservoir.rs:278
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 277 | 2024-04-03T22:19:25.619Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 278 | 2024-04-03T22:19:25.730Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_0c6b0894-9c77-422a-8363-2705812a672c
file = sled-storage/src/dataset.rs:480
|
| 279 | 2024-04-03T22:19:25.738Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
| 280 | 2024-04-03T22:19:25.738Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
| 281 | 2024-04-03T22:19:25.738Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
| 282 | 2024-04-03T22:19:25.738Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 283 | 2024-04-03T22:19:25.738Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:111
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
path = /scratch/u2_2.vdev
|
| 284 | 2024-04-03T22:19:25.748Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev
file = sled-hardware/src/disk.rs:301
|
| 285 | 2024-04-03T22:19:25.748Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:307
id = af9b6346-5d28-46df-baea-640dad9b76c3
|
| 286 | 2024-04-03T22:19:25.841Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }
|
| 287 | 2024-04-03T22:19:25.849Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
| 288 | 2024-04-03T22:19:25.849Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 289 | 2024-04-03T22:19:25.849Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 290 | 2024-04-03T22:19:25.849Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 291 | 2024-04-03T22:19:25.849Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 292 | 2024-04-03T22:19:25.849Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:38
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 293 | 2024-04-03T22:19:25.849Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 294 | 2024-04-03T22:19:25.853Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 295 | 2024-04-03T22:19:25.885Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:57
|
| 296 | 2024-04-03T22:19:25.893Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_af9b6346-5d28-46df-baea-640dad9b76c3/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 297 | 2024-04-03T22:19:26.019Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_af9b6346-5d28-46df-baea-640dad9b76c3
file = sled-storage/src/dataset.rs:480
|
| 298 | 2024-04-03T22:19:26.027Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
| 299 | 2024-04-03T22:19:26.027Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
| 300 | 2024-04-03T22:19:26.027Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
| 301 | 2024-04-03T22:19:26.027Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 302 | 2024-04-03T22:19:26.027Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:111
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
path = /scratch/u2_3.vdev
|
| 303 | 2024-04-03T22:19:26.038Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev
file = sled-hardware/src/disk.rs:301
|
| 304 | 2024-04-03T22:19:26.038Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:307
id = ae4e2e29-ff41-46eb-978b-335ffe5cd37e
|
| 305 | 2024-04-03T22:19:26.132Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }
|
| 306 | 2024-04-03T22:19:26.141Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
| 307 | 2024-04-03T22:19:26.141Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 308 | 2024-04-03T22:19:26.141Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 309 | 2024-04-03T22:19:26.141Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 310 | 2024-04-03T22:19:26.141Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 311 | 2024-04-03T22:19:26.141Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:38
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 312 | 2024-04-03T22:19:26.141Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 313 | 2024-04-03T22:19:26.177Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:57
|
| 314 | 2024-04-03T22:19:26.185Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_ae4e2e29-ff41-46eb-978b-335ffe5cd37e/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 315 | 2024-04-03T22:19:26.311Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_ae4e2e29-ff41-46eb-978b-335ffe5cd37e
file = sled-storage/src/dataset.rs:480
|
| 316 | 2024-04-03T22:19:26.319Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
| 317 | 2024-04-03T22:19:26.319Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
| 318 | 2024-04-03T22:19:26.319Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
| 319 | 2024-04-03T22:19:26.319Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 320 | 2024-04-03T22:19:26.319Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:111
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
path = /scratch/u2_4.vdev
|
| 321 | 2024-04-03T22:19:26.329Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev
file = sled-hardware/src/disk.rs:301
|
| 322 | 2024-04-03T22:19:26.329Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:307
id = 4eb310e9-0d98-4a07-821c-54f35a32dc4a
|
| 323 | 2024-04-03T22:19:26.421Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }
|
| 324 | 2024-04-03T22:19:26.429Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
| 325 | 2024-04-03T22:19:26.429Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 326 | 2024-04-03T22:19:26.429Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 327 | 2024-04-03T22:19:26.429Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 328 | 2024-04-03T22:19:26.429Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 329 | 2024-04-03T22:19:26.430Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:38
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 330 | 2024-04-03T22:19:26.430Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 331 | 2024-04-03T22:19:26.465Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:57
|
| 332 | 2024-04-03T22:19:26.473Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_4eb310e9-0d98-4a07-821c-54f35a32dc4a/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 333 | 2024-04-03T22:19:26.597Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_4eb310e9-0d98-4a07-821c-54f35a32dc4a
file = sled-storage/src/dataset.rs:480
|
| 334 | 2024-04-03T22:19:26.605Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
| 335 | 2024-04-03T22:19:26.605Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
| 336 | 2024-04-03T22:19:26.605Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
| 337 | 2024-04-03T22:19:26.605Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 338 | 2024-04-03T22:19:26.605Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:111
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
path = /scratch/u2_5.vdev
|
| 339 | 2024-04-03T22:19:26.615Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev
file = sled-hardware/src/disk.rs:301
|
| 340 | 2024-04-03T22:19:26.615Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:307
id = 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4
|
| 341 | 2024-04-03T22:19:26.708Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }
|
| 342 | 2024-04-03T22:19:26.716Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
| 343 | 2024-04-03T22:19:26.716Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 344 | 2024-04-03T22:19:26.716Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 345 | 2024-04-03T22:19:26.716Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 346 | 2024-04-03T22:19:26.716Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 347 | 2024-04-03T22:19:26.716Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:38
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 348 | 2024-04-03T22:19:26.716Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 349 | 2024-04-03T22:19:26.753Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:57
|
| 350 | 2024-04-03T22:19:26.761Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_494a3a1c-f913-4e4b-8eec-202ec6c5bfd4/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 351 | 2024-04-03T22:19:26.858Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 352 | 2024-04-03T22:19:26.883Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_494a3a1c-f913-4e4b-8eec-202ec6c5bfd4
file = sled-storage/src/dataset.rs:480
|
| 353 | 2024-04-03T22:19:26.891Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
| 354 | 2024-04-03T22:19:26.891Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
| 355 | 2024-04-03T22:19:26.891Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
| 356 | 2024-04-03T22:19:26.891Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 357 | 2024-04-03T22:19:26.891Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:111
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
path = /scratch/u2_6.vdev
|
| 358 | 2024-04-03T22:19:26.900Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev
file = sled-hardware/src/disk.rs:301
|
| 359 | 2024-04-03T22:19:26.901Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:307
id = 00b9d2d0-59a4-4682-aa18-072c6202c0c8
|
| 360 | 2024-04-03T22:19:26.994Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }
|
| 361 | 2024-04-03T22:19:27.002Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
| 362 | 2024-04-03T22:19:27.002Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 363 | 2024-04-03T22:19:27.002Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 364 | 2024-04-03T22:19:27.002Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 365 | 2024-04-03T22:19:27.002Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 366 | 2024-04-03T22:19:27.003Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:38
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 367 | 2024-04-03T22:19:27.003Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 368 | 2024-04-03T22:19:27.038Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:57
|
| 369 | 2024-04-03T22:19:27.046Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_00b9d2d0-59a4-4682-aa18-072c6202c0c8/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 370 | 2024-04-03T22:19:27.169Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_00b9d2d0-59a4-4682-aa18-072c6202c0c8
file = sled-storage/src/dataset.rs:480
|
| 371 | 2024-04-03T22:19:27.177Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
| 372 | 2024-04-03T22:19:27.177Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
| 373 | 2024-04-03T22:19:27.177Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
| 374 | 2024-04-03T22:19:27.177Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 375 | 2024-04-03T22:19:27.177Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:111
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
path = /scratch/u2_7.vdev
|
| 376 | 2024-04-03T22:19:27.187Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev
file = sled-hardware/src/disk.rs:301
|
| 377 | 2024-04-03T22:19:27.187Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:307
id = 70a16060-4587-476f-9772-16ea3a5a39aa
|
| 378 | 2024-04-03T22:19:27.282Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }
|
| 379 | 2024-04-03T22:19:27.290Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
| 380 | 2024-04-03T22:19:27.290Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 381 | 2024-04-03T22:19:27.290Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 382 | 2024-04-03T22:19:27.290Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 383 | 2024-04-03T22:19:27.290Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 384 | 2024-04-03T22:19:27.290Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:38
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 385 | 2024-04-03T22:19:27.290Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 386 | 2024-04-03T22:19:27.324Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 387 | 2024-04-03T22:19:27.326Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:57
|
| 388 | 2024-04-03T22:19:27.334Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_70a16060-4587-476f-9772-16ea3a5a39aa/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 389 | 2024-04-03T22:19:27.457Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_70a16060-4587-476f-9772-16ea3a5a39aa
file = sled-storage/src/dataset.rs:480
|
| 390 | 2024-04-03T22:19:27.465Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
| 391 | 2024-04-03T22:19:27.465Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
| 392 | 2024-04-03T22:19:27.465Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
| 393 | 2024-04-03T22:19:27.465Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 394 | 2024-04-03T22:19:27.465Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:111
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
path = /scratch/u2_8.vdev
|
| 395 | 2024-04-03T22:19:27.475Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev
file = sled-hardware/src/disk.rs:301
|
| 396 | 2024-04-03T22:19:27.475Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:307
id = 761d3524-c333-4332-9563-7d1ceb36709e
|
| 397 | 2024-04-03T22:19:27.574Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }
|
| 398 | 2024-04-03T22:19:27.584Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
| 399 | 2024-04-03T22:19:27.584Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 400 | 2024-04-03T22:19:27.584Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 401 | 2024-04-03T22:19:27.584Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 402 | 2024-04-03T22:19:27.584Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 403 | 2024-04-03T22:19:27.584Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:38
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 404 | 2024-04-03T22:19:27.584Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 405 | 2024-04-03T22:19:27.624Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:57
|
| 406 | 2024-04-03T22:19:27.632Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_761d3524-c333-4332-9563-7d1ceb36709e/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 407 | 2024-04-03T22:19:27.751Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_761d3524-c333-4332-9563-7d1ceb36709e
file = sled-storage/src/dataset.rs:480
|
| 408 | 2024-04-03T22:19:27.758Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
| 409 | 2024-04-03T22:19:27.758Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
| 410 | 2024-04-03T22:19:27.758Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.omicron-physical-disks.json.tmp
file = common/src/ledger.rs:199
request = omicron_physical_disks_ensure
|
| 411 | 2024-04-03T22:19:27.759Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 412 | 2024-04-03T22:19:27.759Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.omicron-physical-disks.json.tmp
file = common/src/ledger.rs:199
request = omicron_physical_disks_ensure
|
| 413 | 2024-04-03T22:19:27.759Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 12372644
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46916
req_id = ab7b66d3-e652-4720-80f6-8ce157fa6938
response_code = 200
uri = /omicron-physical-disks
|
| 414 | 2024-04-03T22:19:27.790Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 415 | 2024-04-03T22:19:27.790Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:43473
|
| 416 | 2024-04-03T22:19:27.791Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 417 | 2024-04-03T22:19:27.791Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 418 | 2024-04-03T22:19:27.836Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 419 | 2024-04-03T22:19:27.836Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 420 | 2024-04-03T22:19:27.861Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 421 | 2024-04-03T22:19:27.885Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 422 | 2024-04-03T22:19:27.885Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 423 | 2024-04-03T22:19:27.930Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 424 | 2024-04-03T22:19:27.930Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 425 | 2024-04-03T22:19:27.930Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 426 | 2024-04-03T22:19:27.930Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 427 | 2024-04-03T22:19:27.971Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 428 | 2024-04-03T22:19:27.971Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 429 | 2024-04-03T22:19:27.971Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 430 | 2024-04-03T22:19:28.019Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca
file = illumos-utils/src/zone.rs:315
|
| 431 | 2024-04-03T22:19:28.023Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e
file = illumos-utils/src/zone.rs:315
|
| 432 | 2024-04-03T22:19:28.027Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640
file = illumos-utils/src/zone.rs:315
|
| 433 | 2024-04-03T22:19:28.039Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca
file = illumos-utils/src/zone.rs:354
|
| 434 | 2024-04-03T22:19:28.044Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e
file = illumos-utils/src/zone.rs:354
|
| 435 | 2024-04-03T22:19:28.048Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640
file = illumos-utils/src/zone.rs:354
|
| 436 | 2024-04-03T22:19:28.864Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 437 | 2024-04-03T22:19:29.322Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 438 | 2024-04-03T22:19:29.866Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 439 | 2024-04-03T22:19:30.423Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService3"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:2::2"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:2::1"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:2::1]:5353"/>
<propval type="astring" name="dns_address" value="[fd00:1122:3344:2::1]:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 440 | 2024-04-03T22:19:30.423Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }
|
| 441 | 2024-04-03T22:19:30.424Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640
|
| 442 | 2024-04-03T22:19:30.906Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService1"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:3::2"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:3::1"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:3::1]:5353"/>
<propval type="astring" name="dns_address" value="[fd00:1122:3344:3::1]:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 443 | 2024-04-03T22:19:31.387Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }
|
| 444 | 2024-04-03T22:19:31.387Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService2"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:1::2"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:1::1"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:1::1]:5353"/>
<propval type="astring" name="dns_address" value="[fd00:1122:3344:1::1]:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 445 | 2024-04-03T22:19:31.395Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640") failed: Property not found. retry in 43.569159ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640
|
| 446 | 2024-04-03T22:19:31.395Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }
|
| 447 | 2024-04-03T22:19:31.395Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca
|
| 448 | 2024-04-03T22:19:31.400Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e
|
| 449 | 2024-04-03T22:19:31.449Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640") failed: Property not found. retry in 82.662707ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640
|
| 450 | 2024-04-03T22:19:31.521Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca") failed: Property not found. retry in 61.178837ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca
|
| 451 | 2024-04-03T22:19:31.528Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e") failed: Property not found. retry in 72.516622ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e
|
| 452 | 2024-04-03T22:19:31.539Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640") failed: Property not found. retry in 295.895179ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640
|
| 453 | 2024-04-03T22:19:31.638Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca") failed: Property not found. retry in 88.46188ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca
|
| 454 | 2024-04-03T22:19:31.696Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e") failed: Property not found. retry in 81.924751ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e
|
| 455 | 2024-04-03T22:19:31.744Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca") failed: Property not found. retry in 215.930069ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca
|
| 456 | 2024-04-03T22:19:31.835Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e") failed: Property not found. retry in 176.077063ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e
|
| 457 | 2024-04-03T22:19:31.858Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640") failed: Property not found. retry in 444.834572ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640
|
| 458 | 2024-04-03T22:19:31.989Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca") failed: Property not found. retry in 410.230104ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca
|
| 459 | 2024-04-03T22:19:32.023Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e") failed: Property not found. retry in 390.927711ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e
|
| 460 | 2024-04-03T22:19:32.312Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640") failed: Property not found. retry in 1.01788785s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640
|
| 461 | 2024-04-03T22:19:32.408Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca") failed: Property not found. retry in 1.157318256s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca
|
| 462 | 2024-04-03T22:19:32.422Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e") failed: Property not found. retry in 1.038204673s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e
|
| 463 | 2024-04-03T22:19:33.340Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640") failed: Property not found. retry in 768.634957ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640
|
| 464 | 2024-04-03T22:19:33.471Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e") failed: Property not found. retry in 507.757197ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e
|
| 465 | 2024-04-03T22:19:33.576Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca") failed: Property not found. retry in 1.189869056s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca
|
| 466 | 2024-04-03T22:19:33.641Z | INFO | SledAgent (ServiceManager): Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone
file = sled-agent/src/services.rs:2311
|
| 467 | 2024-04-03T22:19:33.641Z | INFO | SledAgent (ServiceManager): Adding bootstrap address
file = illumos-utils/src/running_zone.rs:626
zone = oxz_switch
|
| 468 | 2024-04-03T22:19:33.988Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e") failed: Property not found. retry in 660.040542ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e
|
| 469 | 2024-04-03T22:19:34.117Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640") failed: Property not found. retry in 1.164775827s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640
|
| 470 | 2024-04-03T22:19:34.186Z | INFO | SledAgent (ServiceManager): Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fedf:8a3e
file = sled-agent/src/services.rs:2318
|
| 471 | 2024-04-03T22:19:34.192Z | INFO | SledAgent (ServiceManager): Zone not using gateway (even though underlay is up)
file = sled-agent/src/services.rs:2379
|
| 472 | 2024-04-03T22:19:34.313Z | INFO | SledAgent (ServiceManager): Setting up dendrite service
file = sled-agent/src/services.rs:2564
|
| 473 | 2024-04-03T22:19:34.560Z | INFO | SledAgent (ServiceManager): Setting up lldpd service
file = sled-agent/src/services.rs:2728
|
| 474 | 2024-04-03T22:19:34.658Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e") failed: Property not found. retry in 1.326623155s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e
|
| 475 | 2024-04-03T22:19:34.756Z | INFO | SledAgent (ServiceManager): Setting up MGS service
file = sled-agent/src/services.rs:2443
|
| 476 | 2024-04-03T22:19:34.776Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca") failed: Property not found. retry in 691.374532ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca
|
| 477 | 2024-04-03T22:19:35.055Z | INFO | SledAgent (ServiceManager): Setting up wicketd service
file = sled-agent/src/services.rs:2478
|
| 478 | 2024-04-03T22:19:35.260Z | INFO | SledAgent (ServiceManager): Setting up mgd service
file = sled-agent/src/services.rs:2766
|
| 479 | 2024-04-03T22:19:35.292Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640") failed: Property not found. retry in 658.794292ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640
|
| 480 | 2024-04-03T22:19:35.446Z | INFO | SledAgent (ServiceManager): Setting up mg-ddm service
file = sled-agent/src/services.rs:2793
|
| 481 | 2024-04-03T22:19:35.479Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca") failed: Property not found. retry in 1.236379606s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca
|
| 482 | 2024-04-03T22:19:35.668Z | INFO | SledAgent (ServiceManager): Setting up tfport service
file = sled-agent/src/services.rs:2672
|
| 483 | 2024-04-03T22:19:35.858Z | INFO | SledAgent (ServiceManager): Setting up Simulated SP service
file = sled-agent/src/services.rs:2472
|
| 484 | 2024-04-03T22:19:35.868Z | INFO | SledAgent (ServiceManager): Re-enabling running switch zone (new address)
file = sled-agent/src/services.rs:3851
new = [fd00:1122:3344:101::2, ::1]
old = [::1]
|
| 485 | 2024-04-03T22:19:35.868Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists
file = sled-agent/src/services.rs:3867
|
| 486 | 2024-04-03T22:19:35.868Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:609
zone = oxz_switch
|
| 487 | 2024-04-03T22:19:35.961Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640") failed: Property not found. retry in 893.306269ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640
|
| 488 | 2024-04-03T22:19:35.994Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e") failed: Property not found. retry in 909.619501ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e
|
| 489 | 2024-04-03T22:19:36.423Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists - OK
file = sled-agent/src/services.rs:3875
|
| 490 | 2024-04-03T22:19:36.429Z | INFO | SledAgent (ServiceManager): configuring dendrite service
file = sled-agent/src/services.rs:3931
|
| 491 | 2024-04-03T22:19:36.583Z | INFO | SledAgent (ServiceManager): configuring lldp service
file = sled-agent/src/services.rs:3993
|
| 492 | 2024-04-03T22:19:36.706Z | INFO | SledAgent (ServiceManager): configuring wicketd
file = sled-agent/src/services.rs:3974
rack_subnet = fd00:1122:3344::
|
| 493 | 2024-04-03T22:19:36.735Z | INFO | SledAgent (ServiceManager): configuring mgd service
file = sled-agent/src/services.rs:4016
|
| 494 | 2024-04-03T22:19:36.850Z | INFO | SledAgent (ServiceManager): configuring mg-ddm service
file = sled-agent/src/services.rs:4043
|
| 495 | 2024-04-03T22:19:36.925Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca
|
| 496 | 2024-04-03T22:19:36.925Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e
|
| 497 | 2024-04-03T22:19:36.925Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640
|
| 498 | 2024-04-03T22:19:36.925Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 499 | 2024-04-03T22:19:36.925Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 500 | 2024-04-03T22:19:36.926Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 9135283
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43473
req_id = be30061a-13c2-4fd5-901f-9d8d52d6ffbf
response_code = 204
uri = /omicron-zones
|
| 501 | 2024-04-03T22:19:36.927Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:1::1]:5353
file = sled-agent/src/rack_setup/service.rs:545
|
| 502 | 2024-04-03T22:19:36.979Z | INFO | SledAgent (ServiceManager): Determining physical location of our switch zone at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:364
|
| 503 | 2024-04-03T22:19:37.011Z | INFO | SledAgent (ServiceManager): Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:412
|
| 504 | 2024-04-03T22:19:37.043Z | INFO | SledAgent (ServiceManager): Building Port Configuration
file = sled-agent/src/bootstrap/early_networking.rs:589
|
| 505 | 2024-04-03T22:19:37.043Z | INFO | SledAgent (ServiceManager): Checking dendrite uptime
file = sled-agent/src/bootstrap/early_networking.rs:627
|
| 506 | 2024-04-03T22:19:37.043Z | INFO | SledAgent (ServiceManager): Dendrite online
file = sled-agent/src/bootstrap/early_networking.rs:630
uptime = 3
|
| 507 | 2024-04-03T22:19:37.043Z | INFO | SledAgent (ServiceManager): Configuring default uplink on switch
config = PortSettings {\n links: {\n "0": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}
file = sled-agent/src/bootstrap/early_networking.rs:434
|
| 508 | 2024-04-03T22:19:37.117Z | INFO | SledAgent (StorageManager): Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }
file = sled-storage/src/manager.rs:381
|
| 509 | 2024-04-03T22:19:37.117Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }
|
| 510 | 2024-04-03T22:19:37.117Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
|
| 511 | 2024-04-03T22:19:37.117Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }
|
| 512 | 2024-04-03T22:19:37.117Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
|
| 513 | 2024-04-03T22:19:37.117Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
|
| 514 | 2024-04-03T22:19:37.117Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
|
| 515 | 2024-04-03T22:19:37.117Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
|
| 516 | 2024-04-03T22:19:37.117Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
|
| 517 | 2024-04-03T22:19:37.117Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
|
| 518 | 2024-04-03T22:19:37.117Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
|
| 519 | 2024-04-03T22:19:37.117Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
|
| 520 | 2024-04-03T22:19:37.254Z | WARN | SledAgent (RSS): failed to write DNS configuration (will retry in 144.219177ms)
dns_config_addr = [fd00:1122:3344:1::1]:5353
error_message = Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:565
|
| 521 | 2024-04-03T22:19:37.416Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:2::1]:5353
file = sled-agent/src/rack_setup/service.rs:545
|
| 522 | 2024-04-03T22:19:37.461Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:3::1]:5353
file = sled-agent/src/rack_setup/service.rs:545
|
| 523 | 2024-04-03T22:19:37.507Z | INFO | SledAgent (RSS): Configured all DNS servers
file = sled-agent/src/rack_setup/service.rs:581
|
| 524 | 2024-04-03T22:19:37.507Z | INFO | SledAgent (RSS): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:258
|
| 525 | 2024-04-03T22:19:37.538Z | INFO | SledAgent (RSS): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:275
|
| 526 | 2024-04-03T22:19:37.538Z | INFO | SledAgent (RSS): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:300
|
| 527 | 2024-04-03T22:19:37.538Z | INFO | SledAgent (RSS): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:192
|
| 528 | 2024-04-03T22:19:37.567Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 529 | 2024-04-03T22:19:37.567Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:58502
|
| 530 | 2024-04-03T22:19:37.567Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 531 | 2024-04-03T22:19:37.567Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 532 | 2024-04-03T22:19:37.589Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 533 | 2024-04-03T22:19:37.589Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 534 | 2024-04-03T22:19:37.609Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 535 | 2024-04-03T22:19:37.609Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 536 | 2024-04-03T22:19:37.627Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 537 | 2024-04-03T22:19:37.627Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 538 | 2024-04-03T22:19:37.627Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 539 | 2024-04-03T22:19:37.627Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 540 | 2024-04-03T22:19:37.639Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:258
|
| 541 | 2024-04-03T22:19:37.671Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:275
|
| 542 | 2024-04-03T22:19:37.672Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:300
|
| 543 | 2024-04-03T22:19:37.672Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:192
|
| 544 | 2024-04-03T22:19:37.716Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte0", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 219, 117]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: "vopte0" } }
|
| 545 | 2024-04-03T22:19:37.716Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1285
zone_type = ntp
|
| 546 | 2024-04-03T22:19:37.716Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.1.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 547 | 2024-04-03T22:19:37.716Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.1.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 548 | 2024-04-03T22:19:37.717Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.1.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 549 | 2024-04-03T22:19:37.718Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 550 | 2024-04-03T22:19:37.743Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5
file = illumos-utils/src/zone.rs:315
|
| 551 | 2024-04-03T22:19:37.762Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5
file = illumos-utils/src/zone.rs:354
|
| 552 | 2024-04-03T22:19:38.888Z | INFO | SledAgent (ServiceManager): Profile for oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService4"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::1b"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="1.1.1.1" />
<value_node value="9.9.9.9" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/ntp">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="allow" value="fd00:1122:3344:100::/56"/>
<propval type="boolean" name="boundary" value="true"/>
<propval type="astring" name="server" value="0.pool.ntp.org"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte0"/>
<propval type="astring" name="gateway" value="172.30.3.1"/>
<propval type="astring" name="ip" value="172.30.3.5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 553 | 2024-04-03T22:19:38.889Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5
|
| 554 | 2024-04-03T22:19:39.021Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5") failed: Property not found. retry in 47.933486ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5
|
| 555 | 2024-04-03T22:19:39.077Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5") failed: Property not found. retry in 65.260398ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5
|
| 556 | 2024-04-03T22:19:39.157Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5") failed: Property not found. retry in 170.551358ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5
|
| 557 | 2024-04-03T22:19:39.338Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5") failed: Property not found. retry in 291.99534ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5
|
| 558 | 2024-04-03T22:19:39.639Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5") failed: Property not found. retry in 988.411442ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5
|
| 559 | 2024-04-03T22:19:40.311Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:61080
|
| 560 | 2024-04-03T22:19:40.311Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 82
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:61080
req_id = 9b383a6b-5b90-49b0-8c81-448457580973
response_code = 200
uri = /baseboard
|
| 561 | 2024-04-03T22:19:40.638Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5") failed: Property not found. retry in 824.069415ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5
|
| 562 | 2024-04-03T22:19:41.471Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5") failed: Property not found. retry in 530.070282ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5
|
| 563 | 2024-04-03T22:19:42.012Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5") failed: Property not found. retry in 1.065131445s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5
|
| 564 | 2024-04-03T22:19:43.088Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5") failed: Property not found. retry in 733.522046ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5
|
| 565 | 2024-04-03T22:19:43.843Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5
|
| 566 | 2024-04-03T22:19:43.843Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 567 | 2024-04-03T22:19:43.843Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 568 | 2024-04-03T22:19:43.843Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 6276507
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:58502
req_id = 48d4a961-9db1-4820-887b-62da02488b84
response_code = 204
uri = /omicron-zones
|
| 569 | 2024-04-03T22:19:43.845Z | INFO | SledAgent (RSS): Waiting for rack time synchronization
file = sled-agent/src/rack_setup/service.rs:622
|
| 570 | 2024-04-03T22:19:43.874Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:602
|
| 571 | 2024-04-03T22:19:43.874Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:45639
|
| 572 | 2024-04-03T22:19:43.885Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3582
|
| 573 | 2024-04-03T22:19:43.885Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 10403
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:45639
req_id = 0e661e4d-320f-4e5a-a1dd-47476b94add8
response_code = 500
uri = /timesync
|
| 574 | 2024-04-03T22:19:43.888Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:652
|
| 575 | 2024-04-03T22:19:44.017Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:602
|
| 576 | 2024-04-03T22:19:44.017Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:55364
|
| 577 | 2024-04-03T22:19:48.028Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3582
|
| 578 | 2024-04-03T22:19:48.028Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 4010610
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:55364
req_id = 58abd866-b85a-4f4d-ad65-8f0921c407f8
response_code = 500
uri = /timesync
|
| 579 | 2024-04-03T22:19:48.031Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:652
|
| 580 | 2024-04-03T22:19:48.176Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:602
|
| 581 | 2024-04-03T22:19:48.176Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:43888
|
| 582 | 2024-04-03T22:19:48.185Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3582
|
| 583 | 2024-04-03T22:19:48.185Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 9256
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:43888
req_id = 38551ec0-6345-495e-b03e-542d7787b720
response_code = 500
uri = /timesync
|
| 584 | 2024-04-03T22:19:48.187Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:652
|
| 585 | 2024-04-03T22:19:48.411Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:602
|
| 586 | 2024-04-03T22:19:48.412Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:35752
|
| 587 | 2024-04-03T22:19:48.420Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3582
|
| 588 | 2024-04-03T22:19:48.420Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 8251
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:35752
req_id = 8f4133d3-113e-4943-aac8-4bd6a8c800ca
response_code = 500
uri = /timesync
|
| 589 | 2024-04-03T22:19:48.422Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:652
|
| 590 | 2024-04-03T22:19:48.564Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:602
|
| 591 | 2024-04-03T22:19:48.564Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:51735
|
| 592 | 2024-04-03T22:19:50.572Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 2007981
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:51735
req_id = 2cada7f3-dc19-4275-9dc0-d81764e83c6c
response_code = 200
uri = /timesync
|
| 593 | 2024-04-03T22:19:50.574Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1712182787.540047, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:630
|
| 594 | 2024-04-03T22:19:50.574Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:652
|
| 595 | 2024-04-03T22:19:50.812Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:602
|
| 596 | 2024-04-03T22:19:50.812Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:54801
|
| 597 | 2024-04-03T22:19:50.820Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 8513
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:54801
req_id = c8256890-d0e6-438e-9964-ffd42f1666cc
response_code = 200
uri = /timesync
|
| 598 | 2024-04-03T22:19:50.823Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1712182787.540047, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:630
|
| 599 | 2024-04-03T22:19:50.823Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:652
|
| 600 | 2024-04-03T22:19:51.100Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:602
|
| 601 | 2024-04-03T22:19:51.100Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:51400
|
| 602 | 2024-04-03T22:19:51.108Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 8456
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:51400
req_id = aaf82ed8-8b78-4cf3-805b-d97aab01c903
response_code = 200
uri = /timesync
|
| 603 | 2024-04-03T22:19:51.111Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1712182787.540047, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:630
|
| 604 | 2024-04-03T22:19:51.111Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:652
|
| 605 | 2024-04-03T22:19:51.499Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:602
|
| 606 | 2024-04-03T22:19:51.499Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:35283
|
| 607 | 2024-04-03T22:19:51.507Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 8625
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:35283
req_id = 10d184a1-c51a-4389-bd21-86b484c3c485
response_code = 200
uri = /timesync
|
| 608 | 2024-04-03T22:19:51.509Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1712182787.540047, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:630
|
| 609 | 2024-04-03T22:19:51.509Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:652
|
| 610 | 2024-04-03T22:19:51.914Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:602
|
| 611 | 2024-04-03T22:19:51.914Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:58489
|
| 612 | 2024-04-03T22:19:51.922Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 8366
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:58489
req_id = db58f926-fccb-4d7c-a279-c374ed096d01
response_code = 200
uri = /timesync
|
| 613 | 2024-04-03T22:19:51.924Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1712182787.540047, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:630
|
| 614 | 2024-04-03T22:19:51.924Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:652
|
| 615 | 2024-04-03T22:19:52.265Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:602
|
| 616 | 2024-04-03T22:19:52.265Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:63457
|
| 617 | 2024-04-03T22:19:52.273Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 8226
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:63457
req_id = 7dd78565-0f65-44b9-832e-4a1c40f5e13d
response_code = 200
uri = /timesync
|
| 618 | 2024-04-03T22:19:52.275Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1712182787.540047, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:630
|
| 619 | 2024-04-03T22:19:52.275Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:652
|
| 620 | 2024-04-03T22:19:52.972Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:602
|
| 621 | 2024-04-03T22:19:52.972Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:46614
|
| 622 | 2024-04-03T22:19:52.980Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 8482
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:46614
req_id = 747dc529-024f-4904-badb-0372ed8be49a
response_code = 200
uri = /timesync
|
| 623 | 2024-04-03T22:19:52.982Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1712182787.540047, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:630
|
| 624 | 2024-04-03T22:19:52.982Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:652
|
| 625 | 2024-04-03T22:19:53.174Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:602
|
| 626 | 2024-04-03T22:19:53.174Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:54830
|
| 627 | 2024-04-03T22:19:53.178Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:801
|
| 628 | 2024-04-03T22:19:53.196Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 21540
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:54830
req_id = e3d2c785-edfd-483c-ab1e-6ebdcf650f2e
response_code = 200
uri = /timesync
|
| 629 | 2024-04-03T22:19:53.198Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2728380417, ip_addr: 162.159.200.1, stratum: 4, ref_time: 1712182792.50853, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:630
|
| 630 | 2024-04-03T22:19:53.198Z | INFO | SledAgent (RSS): Finished setting up Internal DNS and NTP
file = sled-agent/src/rack_setup/service.rs:1210
|
| 631 | 2024-04-03T22:19:53.227Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 632 | 2024-04-03T22:19:53.227Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:61922
|
| 633 | 2024-04-03T22:19:53.227Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 634 | 2024-04-03T22:19:53.227Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 635 | 2024-04-03T22:19:53.246Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 636 | 2024-04-03T22:19:53.246Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 637 | 2024-04-03T22:19:53.264Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 638 | 2024-04-03T22:19:53.264Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 639 | 2024-04-03T22:19:53.282Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 640 | 2024-04-03T22:19:53.282Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 641 | 2024-04-03T22:19:53.334Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 642 | 2024-04-03T22:19:53.334Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 643 | 2024-04-03T22:19:53.392Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 644 | 2024-04-03T22:19:53.392Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 645 | 2024-04-03T22:19:53.445Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 646 | 2024-04-03T22:19:53.445Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 647 | 2024-04-03T22:19:53.497Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 648 | 2024-04-03T22:19:53.497Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 649 | 2024-04-03T22:19:53.547Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 650 | 2024-04-03T22:19:53.547Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 651 | 2024-04-03T22:19:53.547Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 652 | 2024-04-03T22:19:53.547Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 653 | 2024-04-03T22:19:53.627Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 654 | 2024-04-03T22:19:53.627Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 655 | 2024-04-03T22:19:53.627Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 656 | 2024-04-03T22:19:53.627Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 657 | 2024-04-03T22:19:53.627Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 658 | 2024-04-03T22:19:53.719Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
file = illumos-utils/src/zone.rs:315
|
| 659 | 2024-04-03T22:19:53.723Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
file = illumos-utils/src/zone.rs:315
|
| 660 | 2024-04-03T22:19:53.727Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
file = illumos-utils/src/zone.rs:315
|
| 661 | 2024-04-03T22:19:53.731Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
file = illumos-utils/src/zone.rs:315
|
| 662 | 2024-04-03T22:19:53.735Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
file = illumos-utils/src/zone.rs:315
|
| 663 | 2024-04-03T22:19:53.740Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
file = illumos-utils/src/zone.rs:354
|
| 664 | 2024-04-03T22:19:53.744Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
file = illumos-utils/src/zone.rs:354
|
| 665 | 2024-04-03T22:19:53.749Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
file = illumos-utils/src/zone.rs:354
|
| 666 | 2024-04-03T22:19:53.753Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
file = illumos-utils/src/zone.rs:354
|
| 667 | 2024-04-03T22:19:53.757Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
file = illumos-utils/src/zone.rs:354
|
| 668 | 2024-04-03T22:19:57.404Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService6"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::3"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::3"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 669 | 2024-04-03T22:19:57.404Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 670 | 2024-04-03T22:19:57.407Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:801
|
| 671 | 2024-04-03T22:19:57.410Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService5"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::5"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 672 | 2024-04-03T22:19:57.411Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 673 | 2024-04-03T22:19:57.418Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService8"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::4"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::4"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 674 | 2024-04-03T22:19:57.419Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
|
| 675 | 2024-04-03T22:19:57.428Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService9"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::7"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::7"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 676 | 2024-04-03T22:19:57.428Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 677 | 2024-04-03T22:19:57.450Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService7"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::6"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::6"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 678 | 2024-04-03T22:19:57.450Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 679 | 2024-04-03T22:19:57.563Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000") failed: Property not found. retry in 64.341634ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 680 | 2024-04-03T22:19:57.570Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac") failed: Property not found. retry in 55.74056ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 681 | 2024-04-03T22:19:57.577Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0") failed: Property not found. retry in 27.462502ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
|
| 682 | 2024-04-03T22:19:57.585Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d") failed: Property not found. retry in 29.025169ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 683 | 2024-04-03T22:19:57.596Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 26.837005ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 684 | 2024-04-03T22:19:57.612Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0") failed: Property not found. retry in 66.841369ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
|
| 685 | 2024-04-03T22:19:57.623Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d") failed: Property not found. retry in 68.536388ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 686 | 2024-04-03T22:19:57.745Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 82.919355ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 687 | 2024-04-03T22:19:57.923Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000") failed: Property not found. retry in 124.278524ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 688 | 2024-04-03T22:19:58.022Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac") failed: Property not found. retry in 82.150206ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 689 | 2024-04-03T22:19:58.249Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0") failed: Property not found. retry in 178.743434ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
|
| 690 | 2024-04-03T22:19:58.521Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d") failed: Property not found. retry in 188.609569ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 691 | 2024-04-03T22:19:58.572Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000") failed: Property not found. retry in 108.572301ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 692 | 2024-04-03T22:19:58.710Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac") failed: Property not found. retry in 217.347288ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 693 | 2024-04-03T22:19:58.789Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 101.994445ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 694 | 2024-04-03T22:19:58.966Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0") failed: Property not found. retry in 426.674006ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
|
| 695 | 2024-04-03T22:19:59.114Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d") failed: Property not found. retry in 298.749914ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 696 | 2024-04-03T22:19:59.218Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000") failed: Property not found. retry in 347.495999ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 697 | 2024-04-03T22:19:59.408Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac") failed: Property not found. retry in 337.819567ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 698 | 2024-04-03T22:19:59.652Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 495.633027ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 699 | 2024-04-03T22:20:00.433Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0") failed: Property not found. retry in 1.171213993s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
|
| 700 | 2024-04-03T22:20:00.506Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d") failed: Property not found. retry in 469.846174ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 701 | 2024-04-03T22:20:00.520Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000") failed: Property not found. retry in 1.059205939s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 702 | 2024-04-03T22:20:00.529Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac") failed: Property not found. retry in 834.93827ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 703 | 2024-04-03T22:20:00.536Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 494.919708ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 704 | 2024-04-03T22:20:00.988Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d") failed: Property not found. retry in 952.391033ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 705 | 2024-04-03T22:20:01.042Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 913.986362ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 706 | 2024-04-03T22:20:01.374Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac") failed: Property not found. retry in 661.042749ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 707 | 2024-04-03T22:20:01.591Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000") failed: Property not found. retry in 506.258606ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 708 | 2024-04-03T22:20:01.614Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0") failed: Property not found. retry in 1.2129297s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
|
| 709 | 2024-04-03T22:20:01.950Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d") failed: Property not found. retry in 746.638021ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 710 | 2024-04-03T22:20:01.967Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 1.142919381s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 711 | 2024-04-03T22:20:02.044Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac") failed: Property not found. retry in 1.169279627s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 712 | 2024-04-03T22:20:02.105Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000") failed: Property not found. retry in 813.106494ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 713 | 2024-04-03T22:20:02.707Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d") failed: Property not found. retry in 885.509721ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 714 | 2024-04-03T22:20:02.836Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0") failed: Property not found. retry in 1.390544729s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
|
| 715 | 2024-04-03T22:20:02.928Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000") failed: Property not found. retry in 1.421226778s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 716 | 2024-04-03T22:20:03.119Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 886.820605ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 717 | 2024-04-03T22:20:03.223Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac") failed: Property not found. retry in 1.129895135s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 718 | 2024-04-03T22:20:03.603Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d") failed: Property not found. retry in 853.875675ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 719 | 2024-04-03T22:20:04.016Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 724.868553ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 720 | 2024-04-03T22:20:04.239Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0") failed: Property not found. retry in 745.313633ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
|
| 721 | 2024-04-03T22:20:04.362Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000") failed: Property not found. retry in 799.527859ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 722 | 2024-04-03T22:20:04.373Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac") failed: Property not found. retry in 606.876483ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 723 | 2024-04-03T22:20:04.470Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d") failed: Property not found. retry in 667.985464ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 724 | 2024-04-03T22:20:04.753Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 842.261845ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 725 | 2024-04-03T22:20:04.989Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac") failed: Property not found. retry in 566.419941ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 726 | 2024-04-03T22:20:04.998Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0") failed: Property not found. retry in 778.64542ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
|
| 727 | 2024-04-03T22:20:05.148Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d") failed: Property not found. retry in 720.361971ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 728 | 2024-04-03T22:20:05.171Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000") failed: Property not found. retry in 511.117183ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 729 | 2024-04-03T22:20:05.566Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac") failed: Property not found. retry in 546.696342ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 730 | 2024-04-03T22:20:05.604Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 801.009013ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 731 | 2024-04-03T22:20:05.690Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000") failed: Property not found. retry in 1.166790533s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 732 | 2024-04-03T22:20:05.785Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0") failed: Property not found. retry in 1.404308789s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
|
| 733 | 2024-04-03T22:20:05.877Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d") failed: Property not found. retry in 1.217974075s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 734 | 2024-04-03T22:20:06.122Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac") failed: Property not found. retry in 1.412488559s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 735 | 2024-04-03T22:20:06.415Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 1.451043598s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 736 | 2024-04-03T22:20:06.866Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000") failed: Property not found. retry in 691.623952ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 737 | 2024-04-03T22:20:07.106Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d") failed: Property not found. retry in 1.47380394s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 738 | 2024-04-03T22:20:07.199Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0") failed: Property not found. retry in 1.338805725s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
|
| 739 | 2024-04-03T22:20:07.544Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac") failed: Property not found. retry in 837.439996ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 740 | 2024-04-03T22:20:07.568Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000") failed: Property not found. retry in 989.067038ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 741 | 2024-04-03T22:20:07.878Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 836.542652ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 742 | 2024-04-03T22:20:08.724Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 980.827202ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 743 | 2024-04-03T22:20:09.716Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 1.383012018s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 744 | 2024-04-03T22:20:09.747Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:34765
|
| 745 | 2024-04-03T22:20:09.747Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 69
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:34765
req_id = a40d465f-7506-4eca-9648-fc59b70a7205
response_code = 200
uri = /baseboard
|
| 746 | 2024-04-03T22:20:11.112Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 957.526483ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 747 | 2024-04-03T22:20:12.133Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 502.597775ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 748 | 2024-04-03T22:20:12.648Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 515.220152ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 749 | 2024-04-03T22:20:13.178Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 716.143635ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 750 | 2024-04-03T22:20:13.904Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 982.407355ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 751 | 2024-04-03T22:20:14.896Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 1.293010308s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 752 | 2024-04-03T22:20:16.199Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 743.316009ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 753 | 2024-04-03T22:20:16.951Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 1.163034171s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 754 | 2024-04-03T22:20:18.127Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5") failed: Property not found. retry in 1.176259619s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 755 | 2024-04-03T22:20:19.324Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000
|
| 756 | 2024-04-03T22:20:19.324Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac
|
| 757 | 2024-04-03T22:20:19.324Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5
|
| 758 | 2024-04-03T22:20:19.324Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0
|
| 759 | 2024-04-03T22:20:19.324Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d
|
| 760 | 2024-04-03T22:20:19.324Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 761 | 2024-04-03T22:20:19.430Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 762 | 2024-04-03T22:20:19.490Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 26166483
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61922
req_id = 28fa9715-c05e-4731-b57b-9304c1ac1bb7
response_code = 204
uri = /omicron-zones
|
| 763 | 2024-04-03T22:20:19.524Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:38484
|
| 764 | 2024-04-03T22:20:19.557Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 33191
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:38484
req_id = d2eff7e6-9008-4714-83ce-28f752365d55
response_code = 500
uri = /cockroachdb
|
| 765 | 2024-04-03T22:20:19.557Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d2eff7e6-9008-4714-83ce-28f752365d55", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:19 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d2eff7e6-9008-4714-83ce-28f752365d55" }
file = sled-agent/src/rack_setup/service.rs:952
retry_after = 76.990711ms
|
| 766 | 2024-04-03T22:20:19.662Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 26667
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:38484
req_id = 43427e4b-7ec2-4231-afa2-b50c8c73959b
response_code = 500
uri = /cockroachdb
|
| 767 | 2024-04-03T22:20:19.663Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "43427e4b-7ec2-4231-afa2-b50c8c73959b", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:19 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "43427e4b-7ec2-4231-afa2-b50c8c73959b" }
file = sled-agent/src/rack_setup/service.rs:952
retry_after = 136.497358ms
|
| 768 | 2024-04-03T22:20:19.824Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 24294
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:38484
req_id = fe40fe15-8708-4183-8fd7-a88809ab44b1
response_code = 500
uri = /cockroachdb
|
| 769 | 2024-04-03T22:20:19.824Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "fe40fe15-8708-4183-8fd7-a88809ab44b1", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:19 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "fe40fe15-8708-4183-8fd7-a88809ab44b1" }
file = sled-agent/src/rack_setup/service.rs:952
retry_after = 113.0573ms
|
| 770 | 2024-04-03T22:20:19.966Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 27086
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:38484
req_id = 105e237d-0be1-4c8c-834c-aa2d34b3c5f1
response_code = 500
uri = /cockroachdb
|
| 771 | 2024-04-03T22:20:19.966Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "105e237d-0be1-4c8c-834c-aa2d34b3c5f1", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:19 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "105e237d-0be1-4c8c-834c-aa2d34b3c5f1" }
file = sled-agent/src/rack_setup/service.rs:952
retry_after = 115.562004ms
|
| 772 | 2024-04-03T22:20:20.110Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 27466
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:38484
req_id = 3ad47092-85f5-45a3-9e88-e60165906be7
response_code = 500
uri = /cockroachdb
|
| 773 | 2024-04-03T22:20:20.111Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3ad47092-85f5-45a3-9e88-e60165906be7", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:19 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3ad47092-85f5-45a3-9e88-e60165906be7" }
file = sled-agent/src/rack_setup/service.rs:952
retry_after = 169.771217ms
|
| 774 | 2024-04-03T22:20:20.308Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 25749
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:38484
req_id = f96e824f-0844-480c-b422-76fb380617a2
response_code = 500
uri = /cockroachdb
|
| 775 | 2024-04-03T22:20:20.308Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f96e824f-0844-480c-b422-76fb380617a2", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:19 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f96e824f-0844-480c-b422-76fb380617a2" }
file = sled-agent/src/rack_setup/service.rs:952
retry_after = 297.120078ms
|
| 776 | 2024-04-03T22:20:20.632Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 26595
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:38484
req_id = 22a7cfda-438c-4b13-a640-18a150fb413d
response_code = 500
uri = /cockroachdb
|
| 777 | 2024-04-03T22:20:20.633Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "22a7cfda-438c-4b13-a640-18a150fb413d", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "22a7cfda-438c-4b13-a640-18a150fb413d" }
file = sled-agent/src/rack_setup/service.rs:952
retry_after = 441.328299ms
|
| 778 | 2024-04-03T22:20:21.103Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 28151
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:38484
req_id = 1db6d45d-eafb-492e-b1ae-f3ab8bf94a33
response_code = 500
uri = /cockroachdb
|
| 779 | 2024-04-03T22:20:21.103Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "1db6d45d-eafb-492e-b1ae-f3ab8bf94a33", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "1db6d45d-eafb-492e-b1ae-f3ab8bf94a33" }
file = sled-agent/src/rack_setup/service.rs:952
retry_after = 422.754449ms
|
| 780 | 2024-04-03T22:20:21.556Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 29201
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:38484
req_id = be675faf-6ca6-486c-9457-eab0e70f0430
response_code = 500
uri = /cockroachdb
|
| 781 | 2024-04-03T22:20:21.556Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "be675faf-6ca6-486c-9457-eab0e70f0430", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "be675faf-6ca6-486c-9457-eab0e70f0430" }
file = sled-agent/src/rack_setup/service.rs:952
retry_after = 640.134791ms
|
| 782 | 2024-04-03T22:20:22.225Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 27617
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:38484
req_id = f30b01c4-8c0d-48ee-9840-186346bb98af
response_code = 500
uri = /cockroachdb
|
| 783 | 2024-04-03T22:20:22.225Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f30b01c4-8c0d-48ee-9840-186346bb98af", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:22 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f30b01c4-8c0d-48ee-9840-186346bb98af" }
file = sled-agent/src/rack_setup/service.rs:952
retry_after = 290.216482ms
|
| 784 | 2024-04-03T22:20:22.544Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 26937
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:38484
req_id = 16a02198-afbc-4c96-861e-7bd50487130f
response_code = 500
uri = /cockroachdb
|
| 785 | 2024-04-03T22:20:22.544Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "16a02198-afbc-4c96-861e-7bd50487130f", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:22 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "16a02198-afbc-4c96-861e-7bd50487130f" }
file = sled-agent/src/rack_setup/service.rs:952
retry_after = 766.791925ms
|
| 786 | 2024-04-03T22:20:23.341Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::6]:32221
file = sled-agent/src/services.rs:3416
|
| 787 | 2024-04-03T22:20:29.134Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3434
|
| 788 | 2024-04-03T22:20:35.157Z | INFO | SledAgent (ServiceManager): Formatting CRDB - Completed
file = sled-agent/src/services.rs:3457
|
| 789 | 2024-04-03T22:20:35.157Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 11812245
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:38484
req_id = 5414fae1-6644-4125-a58d-24e2f3a5f3d7
response_code = 204
uri = /cockroachdb
|
| 790 | 2024-04-03T22:20:35.191Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 791 | 2024-04-03T22:20:35.191Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:53174
|
| 792 | 2024-04-03T22:20:35.191Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 793 | 2024-04-03T22:20:35.191Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 794 | 2024-04-03T22:20:35.215Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 795 | 2024-04-03T22:20:35.215Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 796 | 2024-04-03T22:20:35.235Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 797 | 2024-04-03T22:20:35.235Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 798 | 2024-04-03T22:20:35.256Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 799 | 2024-04-03T22:20:35.256Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 800 | 2024-04-03T22:20:35.274Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 801 | 2024-04-03T22:20:35.274Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 802 | 2024-04-03T22:20:35.293Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 803 | 2024-04-03T22:20:35.293Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 804 | 2024-04-03T22:20:35.313Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 805 | 2024-04-03T22:20:35.313Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 806 | 2024-04-03T22:20:35.332Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 807 | 2024-04-03T22:20:35.332Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 808 | 2024-04-03T22:20:35.350Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 809 | 2024-04-03T22:20:35.350Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 810 | 2024-04-03T22:20:35.450Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 811 | 2024-04-03T22:20:35.450Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 812 | 2024-04-03T22:20:35.553Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 813 | 2024-04-03T22:20:35.553Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 814 | 2024-04-03T22:20:35.664Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 815 | 2024-04-03T22:20:35.664Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 816 | 2024-04-03T22:20:35.763Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 817 | 2024-04-03T22:20:35.763Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 818 | 2024-04-03T22:20:35.823Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 819 | 2024-04-03T22:20:35.823Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 820 | 2024-04-03T22:20:35.899Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 821 | 2024-04-03T22:20:35.899Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 822 | 2024-04-03T22:20:35.987Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 823 | 2024-04-03T22:20:35.987Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 824 | 2024-04-03T22:20:36.059Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 825 | 2024-04-03T22:20:36.059Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 826 | 2024-04-03T22:20:36.120Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 827 | 2024-04-03T22:20:36.120Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 828 | 2024-04-03T22:20:36.196Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 829 | 2024-04-03T22:20:36.196Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 830 | 2024-04-03T22:20:36.260Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 831 | 2024-04-03T22:20:36.260Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 832 | 2024-04-03T22:20:36.340Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 833 | 2024-04-03T22:20:36.340Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 834 | 2024-04-03T22:20:36.341Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 835 | 2024-04-03T22:20:36.341Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 836 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 837 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 838 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 839 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 840 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 841 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 842 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 843 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 844 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 845 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 846 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 847 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 848 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 849 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 850 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 851 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 852 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 853 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 854 | 2024-04-03T22:20:36.438Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 855 | 2024-04-03T22:20:36.439Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 1244399
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 6bba5f07-5399-4b2b-8300-fd03ade49d0d
response_code = 500
uri = /omicron-zones
|
| 856 | 2024-04-03T22:20:36.439Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6bba5f07-5399-4b2b-8300-fd03ade49d0d", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:36 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6bba5f07-5399-4b2b-8300-fd03ade49d0d" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 90.150003ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 857 | 2024-04-03T22:20:36.530Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 858 | 2024-04-03T22:20:36.531Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 859 | 2024-04-03T22:20:36.531Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 860 | 2024-04-03T22:20:36.551Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 861 | 2024-04-03T22:20:36.551Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 862 | 2024-04-03T22:20:36.573Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 863 | 2024-04-03T22:20:36.573Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 864 | 2024-04-03T22:20:36.591Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 865 | 2024-04-03T22:20:36.591Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 866 | 2024-04-03T22:20:36.611Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 867 | 2024-04-03T22:20:36.611Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 868 | 2024-04-03T22:20:36.629Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 869 | 2024-04-03T22:20:36.629Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 870 | 2024-04-03T22:20:36.646Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 871 | 2024-04-03T22:20:36.646Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 872 | 2024-04-03T22:20:36.664Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 873 | 2024-04-03T22:20:36.664Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 874 | 2024-04-03T22:20:36.684Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 875 | 2024-04-03T22:20:36.684Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 876 | 2024-04-03T22:20:36.702Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 877 | 2024-04-03T22:20:36.702Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 878 | 2024-04-03T22:20:36.722Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 879 | 2024-04-03T22:20:36.722Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 880 | 2024-04-03T22:20:36.742Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 881 | 2024-04-03T22:20:36.742Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 882 | 2024-04-03T22:20:36.761Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 883 | 2024-04-03T22:20:36.761Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 884 | 2024-04-03T22:20:36.780Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 885 | 2024-04-03T22:20:36.780Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 886 | 2024-04-03T22:20:36.803Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 887 | 2024-04-03T22:20:36.803Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 888 | 2024-04-03T22:20:36.822Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 889 | 2024-04-03T22:20:36.822Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 890 | 2024-04-03T22:20:36.841Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 891 | 2024-04-03T22:20:36.841Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 892 | 2024-04-03T22:20:36.861Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 893 | 2024-04-03T22:20:36.861Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 894 | 2024-04-03T22:20:36.881Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 895 | 2024-04-03T22:20:36.881Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 896 | 2024-04-03T22:20:36.900Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 897 | 2024-04-03T22:20:36.900Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 898 | 2024-04-03T22:20:36.920Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 899 | 2024-04-03T22:20:36.920Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 900 | 2024-04-03T22:20:36.920Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 901 | 2024-04-03T22:20:36.921Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 902 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 903 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 904 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 905 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 906 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 907 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 908 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 909 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 910 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 911 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 912 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 913 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 914 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 915 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 916 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 917 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 918 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 919 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 920 | 2024-04-03T22:20:37.015Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 921 | 2024-04-03T22:20:37.016Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 483732
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = ee07c6d8-c3ec-4364-a71e-ab7a12d71ee2
response_code = 500
uri = /omicron-zones
|
| 922 | 2024-04-03T22:20:37.016Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "ee07c6d8-c3ec-4364-a71e-ab7a12d71ee2", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:36 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "ee07c6d8-c3ec-4364-a71e-ab7a12d71ee2" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 161.443097ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 923 | 2024-04-03T22:20:37.180Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 924 | 2024-04-03T22:20:37.181Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 925 | 2024-04-03T22:20:37.181Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 926 | 2024-04-03T22:20:37.203Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 927 | 2024-04-03T22:20:37.203Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 928 | 2024-04-03T22:20:37.226Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 929 | 2024-04-03T22:20:37.226Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 930 | 2024-04-03T22:20:37.248Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 931 | 2024-04-03T22:20:37.248Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 932 | 2024-04-03T22:20:37.267Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 933 | 2024-04-03T22:20:37.267Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 934 | 2024-04-03T22:20:37.285Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 935 | 2024-04-03T22:20:37.285Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 936 | 2024-04-03T22:20:37.304Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 937 | 2024-04-03T22:20:37.304Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 938 | 2024-04-03T22:20:37.325Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 939 | 2024-04-03T22:20:37.325Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 940 | 2024-04-03T22:20:37.344Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 941 | 2024-04-03T22:20:37.344Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 942 | 2024-04-03T22:20:37.365Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 943 | 2024-04-03T22:20:37.365Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 944 | 2024-04-03T22:20:37.384Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 945 | 2024-04-03T22:20:37.384Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 946 | 2024-04-03T22:20:37.403Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 947 | 2024-04-03T22:20:37.403Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 948 | 2024-04-03T22:20:37.422Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 949 | 2024-04-03T22:20:37.422Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 950 | 2024-04-03T22:20:37.440Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 951 | 2024-04-03T22:20:37.440Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 952 | 2024-04-03T22:20:37.458Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 953 | 2024-04-03T22:20:37.458Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 954 | 2024-04-03T22:20:37.477Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 955 | 2024-04-03T22:20:37.477Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 956 | 2024-04-03T22:20:37.496Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 957 | 2024-04-03T22:20:37.496Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 958 | 2024-04-03T22:20:37.518Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 959 | 2024-04-03T22:20:37.518Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 960 | 2024-04-03T22:20:37.537Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 961 | 2024-04-03T22:20:37.537Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 962 | 2024-04-03T22:20:37.557Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 963 | 2024-04-03T22:20:37.557Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 964 | 2024-04-03T22:20:37.576Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 965 | 2024-04-03T22:20:37.576Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 966 | 2024-04-03T22:20:37.576Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 967 | 2024-04-03T22:20:37.576Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 968 | 2024-04-03T22:20:37.672Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 969 | 2024-04-03T22:20:37.672Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 970 | 2024-04-03T22:20:37.672Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 971 | 2024-04-03T22:20:37.672Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 972 | 2024-04-03T22:20:37.672Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 973 | 2024-04-03T22:20:37.672Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 974 | 2024-04-03T22:20:37.672Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 975 | 2024-04-03T22:20:37.672Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 976 | 2024-04-03T22:20:37.672Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 977 | 2024-04-03T22:20:37.672Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 978 | 2024-04-03T22:20:37.672Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 979 | 2024-04-03T22:20:37.672Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 980 | 2024-04-03T22:20:37.672Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 981 | 2024-04-03T22:20:37.672Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 982 | 2024-04-03T22:20:37.673Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 983 | 2024-04-03T22:20:37.673Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 984 | 2024-04-03T22:20:37.673Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 985 | 2024-04-03T22:20:37.673Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 986 | 2024-04-03T22:20:37.673Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 987 | 2024-04-03T22:20:37.673Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 491452
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 5377c6a5-8700-4b1c-8dc1-9049c2fd2b72
response_code = 500
uri = /omicron-zones
|
| 988 | 2024-04-03T22:20:37.673Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "5377c6a5-8700-4b1c-8dc1-9049c2fd2b72", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:37 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "5377c6a5-8700-4b1c-8dc1-9049c2fd2b72" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 121.192199ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 989 | 2024-04-03T22:20:37.796Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 990 | 2024-04-03T22:20:37.796Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 991 | 2024-04-03T22:20:37.796Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 992 | 2024-04-03T22:20:37.819Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 993 | 2024-04-03T22:20:37.819Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 994 | 2024-04-03T22:20:37.840Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 995 | 2024-04-03T22:20:37.840Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 996 | 2024-04-03T22:20:37.861Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 997 | 2024-04-03T22:20:37.861Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 998 | 2024-04-03T22:20:37.880Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 999 | 2024-04-03T22:20:37.880Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1000 | 2024-04-03T22:20:37.901Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1001 | 2024-04-03T22:20:37.901Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1002 | 2024-04-03T22:20:37.920Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1003 | 2024-04-03T22:20:37.920Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1004 | 2024-04-03T22:20:37.940Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1005 | 2024-04-03T22:20:37.940Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1006 | 2024-04-03T22:20:37.958Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1007 | 2024-04-03T22:20:37.958Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1008 | 2024-04-03T22:20:37.978Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1009 | 2024-04-03T22:20:37.978Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1010 | 2024-04-03T22:20:37.999Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1011 | 2024-04-03T22:20:37.999Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1012 | 2024-04-03T22:20:38.021Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1013 | 2024-04-03T22:20:38.021Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1014 | 2024-04-03T22:20:38.046Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1015 | 2024-04-03T22:20:38.046Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1016 | 2024-04-03T22:20:38.067Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1017 | 2024-04-03T22:20:38.067Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1018 | 2024-04-03T22:20:38.088Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1019 | 2024-04-03T22:20:38.088Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1020 | 2024-04-03T22:20:38.107Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1021 | 2024-04-03T22:20:38.107Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1022 | 2024-04-03T22:20:38.127Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1023 | 2024-04-03T22:20:38.127Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1024 | 2024-04-03T22:20:38.147Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1025 | 2024-04-03T22:20:38.147Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1026 | 2024-04-03T22:20:38.173Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1027 | 2024-04-03T22:20:38.173Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1028 | 2024-04-03T22:20:38.194Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1029 | 2024-04-03T22:20:38.194Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1030 | 2024-04-03T22:20:38.216Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1031 | 2024-04-03T22:20:38.216Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1032 | 2024-04-03T22:20:38.216Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1033 | 2024-04-03T22:20:38.216Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1034 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1035 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1036 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1037 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1038 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1039 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1040 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1041 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1042 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1043 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1044 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1045 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1046 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1047 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1048 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1049 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1050 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1051 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1052 | 2024-04-03T22:20:38.316Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1053 | 2024-04-03T22:20:38.316Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 518992
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 1573eeb3-1a29-4968-a102-e093b80b2164
response_code = 500
uri = /omicron-zones
|
| 1054 | 2024-04-03T22:20:38.317Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "1573eeb3-1a29-4968-a102-e093b80b2164", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:38 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "1573eeb3-1a29-4968-a102-e093b80b2164" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 88.06165ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1055 | 2024-04-03T22:20:38.406Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1056 | 2024-04-03T22:20:38.407Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1057 | 2024-04-03T22:20:38.407Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1058 | 2024-04-03T22:20:38.431Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1059 | 2024-04-03T22:20:38.431Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1060 | 2024-04-03T22:20:38.453Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1061 | 2024-04-03T22:20:38.453Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1062 | 2024-04-03T22:20:38.475Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1063 | 2024-04-03T22:20:38.475Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1064 | 2024-04-03T22:20:38.498Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1065 | 2024-04-03T22:20:38.498Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1066 | 2024-04-03T22:20:38.519Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1067 | 2024-04-03T22:20:38.519Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1068 | 2024-04-03T22:20:38.541Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1069 | 2024-04-03T22:20:38.541Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1070 | 2024-04-03T22:20:38.562Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1071 | 2024-04-03T22:20:38.562Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1072 | 2024-04-03T22:20:38.583Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1073 | 2024-04-03T22:20:38.583Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1074 | 2024-04-03T22:20:38.606Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1075 | 2024-04-03T22:20:38.607Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1076 | 2024-04-03T22:20:38.626Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1077 | 2024-04-03T22:20:38.626Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1078 | 2024-04-03T22:20:38.648Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1079 | 2024-04-03T22:20:38.648Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1080 | 2024-04-03T22:20:38.671Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1081 | 2024-04-03T22:20:38.671Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1082 | 2024-04-03T22:20:38.692Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1083 | 2024-04-03T22:20:38.692Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1084 | 2024-04-03T22:20:38.718Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1085 | 2024-04-03T22:20:38.718Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1086 | 2024-04-03T22:20:38.743Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1087 | 2024-04-03T22:20:38.744Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1088 | 2024-04-03T22:20:38.767Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1089 | 2024-04-03T22:20:38.767Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1090 | 2024-04-03T22:20:38.793Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1091 | 2024-04-03T22:20:38.793Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1092 | 2024-04-03T22:20:38.818Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1093 | 2024-04-03T22:20:38.818Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1094 | 2024-04-03T22:20:38.844Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1095 | 2024-04-03T22:20:38.844Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1096 | 2024-04-03T22:20:38.869Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1097 | 2024-04-03T22:20:38.870Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1098 | 2024-04-03T22:20:38.870Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1099 | 2024-04-03T22:20:38.870Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1100 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1101 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1102 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1103 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1104 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1105 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1106 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1107 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1108 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1109 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1110 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1111 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1112 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1113 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1114 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1115 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1116 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1117 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1118 | 2024-04-03T22:20:38.985Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1119 | 2024-04-03T22:20:38.985Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 577443
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = cba19c94-b9d7-421f-abf7-9cb6b796fb85
response_code = 500
uri = /omicron-zones
|
| 1120 | 2024-04-03T22:20:38.986Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "cba19c94-b9d7-421f-abf7-9cb6b796fb85", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:38 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "cba19c94-b9d7-421f-abf7-9cb6b796fb85" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 260.919349ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1121 | 2024-04-03T22:20:39.249Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1122 | 2024-04-03T22:20:39.249Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1123 | 2024-04-03T22:20:39.249Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1124 | 2024-04-03T22:20:39.272Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1125 | 2024-04-03T22:20:39.272Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1126 | 2024-04-03T22:20:39.295Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1127 | 2024-04-03T22:20:39.295Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1128 | 2024-04-03T22:20:39.315Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1129 | 2024-04-03T22:20:39.315Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1130 | 2024-04-03T22:20:39.336Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1131 | 2024-04-03T22:20:39.336Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1132 | 2024-04-03T22:20:39.355Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1133 | 2024-04-03T22:20:39.355Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1134 | 2024-04-03T22:20:39.374Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1135 | 2024-04-03T22:20:39.374Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1136 | 2024-04-03T22:20:39.394Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1137 | 2024-04-03T22:20:39.394Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1138 | 2024-04-03T22:20:39.415Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1139 | 2024-04-03T22:20:39.415Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1140 | 2024-04-03T22:20:39.435Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1141 | 2024-04-03T22:20:39.435Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1142 | 2024-04-03T22:20:39.455Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1143 | 2024-04-03T22:20:39.455Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1144 | 2024-04-03T22:20:39.474Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1145 | 2024-04-03T22:20:39.474Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1146 | 2024-04-03T22:20:39.494Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1147 | 2024-04-03T22:20:39.494Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1148 | 2024-04-03T22:20:39.515Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1149 | 2024-04-03T22:20:39.515Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1150 | 2024-04-03T22:20:39.533Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1151 | 2024-04-03T22:20:39.533Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1152 | 2024-04-03T22:20:39.552Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1153 | 2024-04-03T22:20:39.552Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1154 | 2024-04-03T22:20:39.572Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1155 | 2024-04-03T22:20:39.572Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1156 | 2024-04-03T22:20:39.590Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1157 | 2024-04-03T22:20:39.590Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1158 | 2024-04-03T22:20:39.610Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1159 | 2024-04-03T22:20:39.610Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1160 | 2024-04-03T22:20:39.632Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1161 | 2024-04-03T22:20:39.632Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1162 | 2024-04-03T22:20:39.651Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1163 | 2024-04-03T22:20:39.652Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1164 | 2024-04-03T22:20:39.652Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1165 | 2024-04-03T22:20:39.652Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1166 | 2024-04-03T22:20:39.748Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1167 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1168 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1169 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1170 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1171 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1172 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1173 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1174 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1175 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1176 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1177 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1178 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1179 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1180 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1181 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1182 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1183 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1184 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1185 | 2024-04-03T22:20:39.749Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 498457
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = d521b06c-6b9c-4bc6-bc25-81fe7897a223
response_code = 500
uri = /omicron-zones
|
| 1186 | 2024-04-03T22:20:39.749Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d521b06c-6b9c-4bc6-bc25-81fe7897a223", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:38 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d521b06c-6b9c-4bc6-bc25-81fe7897a223" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 190.331832ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1187 | 2024-04-03T22:20:39.822Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:55661
|
| 1188 | 2024-04-03T22:20:39.822Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 40
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:55661
req_id = 8b559960-9817-47c7-b65e-7365ba36c980
response_code = 200
uri = /baseboard
|
| 1189 | 2024-04-03T22:20:39.942Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1190 | 2024-04-03T22:20:39.943Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1191 | 2024-04-03T22:20:39.943Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1192 | 2024-04-03T22:20:39.964Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1193 | 2024-04-03T22:20:39.964Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1194 | 2024-04-03T22:20:39.984Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1195 | 2024-04-03T22:20:39.984Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1196 | 2024-04-03T22:20:40.003Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1197 | 2024-04-03T22:20:40.003Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1198 | 2024-04-03T22:20:40.024Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1199 | 2024-04-03T22:20:40.024Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1200 | 2024-04-03T22:20:40.043Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1201 | 2024-04-03T22:20:40.043Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1202 | 2024-04-03T22:20:40.062Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1203 | 2024-04-03T22:20:40.063Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1204 | 2024-04-03T22:20:40.085Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1205 | 2024-04-03T22:20:40.085Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1206 | 2024-04-03T22:20:40.105Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1207 | 2024-04-03T22:20:40.105Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1208 | 2024-04-03T22:20:40.123Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1209 | 2024-04-03T22:20:40.123Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1210 | 2024-04-03T22:20:40.143Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1211 | 2024-04-03T22:20:40.143Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1212 | 2024-04-03T22:20:40.161Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1213 | 2024-04-03T22:20:40.161Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1214 | 2024-04-03T22:20:40.180Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1215 | 2024-04-03T22:20:40.180Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1216 | 2024-04-03T22:20:40.200Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1217 | 2024-04-03T22:20:40.200Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1218 | 2024-04-03T22:20:40.219Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1219 | 2024-04-03T22:20:40.219Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1220 | 2024-04-03T22:20:40.238Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1221 | 2024-04-03T22:20:40.238Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1222 | 2024-04-03T22:20:40.258Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1223 | 2024-04-03T22:20:40.258Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1224 | 2024-04-03T22:20:40.277Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1225 | 2024-04-03T22:20:40.277Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1226 | 2024-04-03T22:20:40.295Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1227 | 2024-04-03T22:20:40.295Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1228 | 2024-04-03T22:20:40.316Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1229 | 2024-04-03T22:20:40.316Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1230 | 2024-04-03T22:20:40.336Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1231 | 2024-04-03T22:20:40.336Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1232 | 2024-04-03T22:20:40.336Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1233 | 2024-04-03T22:20:40.336Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1234 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1235 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1236 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1237 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1238 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1239 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1240 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1241 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1242 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1243 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1244 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1245 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1246 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1247 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1248 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1249 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1250 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1251 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1252 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1253 | 2024-04-03T22:20:40.432Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 488479
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = f51d49ff-dfb0-4048-a9a2-ba3a5834d363
response_code = 500
uri = /omicron-zones
|
| 1254 | 2024-04-03T22:20:40.432Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f51d49ff-dfb0-4048-a9a2-ba3a5834d363", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:39 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f51d49ff-dfb0-4048-a9a2-ba3a5834d363" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 214.23252ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1255 | 2024-04-03T22:20:40.648Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1256 | 2024-04-03T22:20:40.649Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1257 | 2024-04-03T22:20:40.649Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1258 | 2024-04-03T22:20:40.671Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1259 | 2024-04-03T22:20:40.671Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1260 | 2024-04-03T22:20:40.693Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1261 | 2024-04-03T22:20:40.693Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1262 | 2024-04-03T22:20:40.714Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1263 | 2024-04-03T22:20:40.714Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1264 | 2024-04-03T22:20:40.733Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1265 | 2024-04-03T22:20:40.733Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1266 | 2024-04-03T22:20:40.751Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1267 | 2024-04-03T22:20:40.751Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1268 | 2024-04-03T22:20:40.771Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1269 | 2024-04-03T22:20:40.771Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1270 | 2024-04-03T22:20:40.790Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1271 | 2024-04-03T22:20:40.790Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1272 | 2024-04-03T22:20:40.809Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1273 | 2024-04-03T22:20:40.809Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1274 | 2024-04-03T22:20:40.830Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1275 | 2024-04-03T22:20:40.830Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1276 | 2024-04-03T22:20:40.851Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1277 | 2024-04-03T22:20:40.851Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1278 | 2024-04-03T22:20:40.872Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1279 | 2024-04-03T22:20:40.873Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1280 | 2024-04-03T22:20:40.893Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1281 | 2024-04-03T22:20:40.893Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1282 | 2024-04-03T22:20:40.914Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1283 | 2024-04-03T22:20:40.914Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1284 | 2024-04-03T22:20:40.934Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1285 | 2024-04-03T22:20:40.934Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1286 | 2024-04-03T22:20:40.953Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1287 | 2024-04-03T22:20:40.954Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1288 | 2024-04-03T22:20:40.974Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1289 | 2024-04-03T22:20:40.974Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1290 | 2024-04-03T22:20:40.994Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1291 | 2024-04-03T22:20:40.994Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1292 | 2024-04-03T22:20:41.016Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1293 | 2024-04-03T22:20:41.016Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1294 | 2024-04-03T22:20:41.037Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1295 | 2024-04-03T22:20:41.037Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1296 | 2024-04-03T22:20:41.056Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1297 | 2024-04-03T22:20:41.056Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1298 | 2024-04-03T22:20:41.056Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1299 | 2024-04-03T22:20:41.056Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1300 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1301 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1302 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1303 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1304 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1305 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1306 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1307 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1308 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1309 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1310 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1311 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1312 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1313 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1314 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1315 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1316 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1317 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1318 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1319 | 2024-04-03T22:20:41.156Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 506219
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 12118638-ae3a-49e1-ac6e-1da7ea919cc4
response_code = 500
uri = /omicron-zones
|
| 1320 | 2024-04-03T22:20:41.156Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "12118638-ae3a-49e1-ac6e-1da7ea919cc4", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:41 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "12118638-ae3a-49e1-ac6e-1da7ea919cc4" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 354.975756ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1321 | 2024-04-03T22:20:41.514Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1322 | 2024-04-03T22:20:41.515Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1323 | 2024-04-03T22:20:41.515Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1324 | 2024-04-03T22:20:41.539Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1325 | 2024-04-03T22:20:41.539Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1326 | 2024-04-03T22:20:41.560Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1327 | 2024-04-03T22:20:41.560Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1328 | 2024-04-03T22:20:41.580Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1329 | 2024-04-03T22:20:41.580Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1330 | 2024-04-03T22:20:41.599Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1331 | 2024-04-03T22:20:41.599Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1332 | 2024-04-03T22:20:41.618Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1333 | 2024-04-03T22:20:41.618Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1334 | 2024-04-03T22:20:41.637Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1335 | 2024-04-03T22:20:41.637Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1336 | 2024-04-03T22:20:41.657Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1337 | 2024-04-03T22:20:41.657Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1338 | 2024-04-03T22:20:41.676Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1339 | 2024-04-03T22:20:41.676Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1340 | 2024-04-03T22:20:41.695Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1341 | 2024-04-03T22:20:41.695Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1342 | 2024-04-03T22:20:41.716Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1343 | 2024-04-03T22:20:41.716Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1344 | 2024-04-03T22:20:41.735Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1345 | 2024-04-03T22:20:41.735Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1346 | 2024-04-03T22:20:41.756Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1347 | 2024-04-03T22:20:41.756Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1348 | 2024-04-03T22:20:41.776Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1349 | 2024-04-03T22:20:41.776Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1350 | 2024-04-03T22:20:41.795Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1351 | 2024-04-03T22:20:41.795Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1352 | 2024-04-03T22:20:41.814Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1353 | 2024-04-03T22:20:41.814Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1354 | 2024-04-03T22:20:41.833Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1355 | 2024-04-03T22:20:41.833Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1356 | 2024-04-03T22:20:41.853Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1357 | 2024-04-03T22:20:41.853Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1358 | 2024-04-03T22:20:41.873Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1359 | 2024-04-03T22:20:41.873Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1360 | 2024-04-03T22:20:41.894Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1361 | 2024-04-03T22:20:41.894Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1362 | 2024-04-03T22:20:41.914Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1363 | 2024-04-03T22:20:41.914Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1364 | 2024-04-03T22:20:41.914Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1365 | 2024-04-03T22:20:41.914Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1366 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1367 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1368 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1369 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1370 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1371 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1372 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1373 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1374 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1375 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1376 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1377 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1378 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1379 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1380 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1381 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1382 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1383 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1384 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1385 | 2024-04-03T22:20:42.012Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 496574
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 39a0c8d4-0f14-4e64-a0fe-79c92d759ef3
response_code = 500
uri = /omicron-zones
|
| 1386 | 2024-04-03T22:20:42.012Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "39a0c8d4-0f14-4e64-a0fe-79c92d759ef3", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:42 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "39a0c8d4-0f14-4e64-a0fe-79c92d759ef3" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 285.090275ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1387 | 2024-04-03T22:20:42.299Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1388 | 2024-04-03T22:20:42.300Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1389 | 2024-04-03T22:20:42.300Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1390 | 2024-04-03T22:20:42.323Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1391 | 2024-04-03T22:20:42.323Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1392 | 2024-04-03T22:20:42.344Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1393 | 2024-04-03T22:20:42.344Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1394 | 2024-04-03T22:20:42.367Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1395 | 2024-04-03T22:20:42.367Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1396 | 2024-04-03T22:20:42.386Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1397 | 2024-04-03T22:20:42.387Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1398 | 2024-04-03T22:20:42.407Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1399 | 2024-04-03T22:20:42.407Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1400 | 2024-04-03T22:20:42.429Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1401 | 2024-04-03T22:20:42.429Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1402 | 2024-04-03T22:20:42.449Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1403 | 2024-04-03T22:20:42.449Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1404 | 2024-04-03T22:20:42.467Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1405 | 2024-04-03T22:20:42.467Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1406 | 2024-04-03T22:20:42.488Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1407 | 2024-04-03T22:20:42.488Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1408 | 2024-04-03T22:20:42.506Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1409 | 2024-04-03T22:20:42.506Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1410 | 2024-04-03T22:20:42.527Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1411 | 2024-04-03T22:20:42.527Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1412 | 2024-04-03T22:20:42.548Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1413 | 2024-04-03T22:20:42.548Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1414 | 2024-04-03T22:20:42.568Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1415 | 2024-04-03T22:20:42.568Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1416 | 2024-04-03T22:20:42.587Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1417 | 2024-04-03T22:20:42.587Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1418 | 2024-04-03T22:20:42.606Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1419 | 2024-04-03T22:20:42.606Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1420 | 2024-04-03T22:20:42.626Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1421 | 2024-04-03T22:20:42.626Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1422 | 2024-04-03T22:20:42.646Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1423 | 2024-04-03T22:20:42.646Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1424 | 2024-04-03T22:20:42.664Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1425 | 2024-04-03T22:20:42.665Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1426 | 2024-04-03T22:20:42.683Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1427 | 2024-04-03T22:20:42.683Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1428 | 2024-04-03T22:20:42.703Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1429 | 2024-04-03T22:20:42.703Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1430 | 2024-04-03T22:20:42.703Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1431 | 2024-04-03T22:20:42.703Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1432 | 2024-04-03T22:20:42.797Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1433 | 2024-04-03T22:20:42.797Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1434 | 2024-04-03T22:20:42.797Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1435 | 2024-04-03T22:20:42.797Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1436 | 2024-04-03T22:20:42.797Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1437 | 2024-04-03T22:20:42.797Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1438 | 2024-04-03T22:20:42.797Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1439 | 2024-04-03T22:20:42.797Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1440 | 2024-04-03T22:20:42.797Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1441 | 2024-04-03T22:20:42.797Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1442 | 2024-04-03T22:20:42.797Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1443 | 2024-04-03T22:20:42.797Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1444 | 2024-04-03T22:20:42.798Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1445 | 2024-04-03T22:20:42.798Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1446 | 2024-04-03T22:20:42.798Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1447 | 2024-04-03T22:20:42.798Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1448 | 2024-04-03T22:20:42.798Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1449 | 2024-04-03T22:20:42.798Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1450 | 2024-04-03T22:20:42.798Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1451 | 2024-04-03T22:20:42.798Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 496814
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = c4074261-2e9e-4c02-bde9-829cc1c88135
response_code = 500
uri = /omicron-zones
|
| 1452 | 2024-04-03T22:20:42.798Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "c4074261-2e9e-4c02-bde9-829cc1c88135", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:42 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "c4074261-2e9e-4c02-bde9-829cc1c88135" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 357.16682ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1453 | 2024-04-03T22:20:43.157Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1454 | 2024-04-03T22:20:43.158Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1455 | 2024-04-03T22:20:43.158Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1456 | 2024-04-03T22:20:43.182Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1457 | 2024-04-03T22:20:43.182Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1458 | 2024-04-03T22:20:43.203Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1459 | 2024-04-03T22:20:43.203Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1460 | 2024-04-03T22:20:43.224Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1461 | 2024-04-03T22:20:43.224Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1462 | 2024-04-03T22:20:43.244Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1463 | 2024-04-03T22:20:43.244Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1464 | 2024-04-03T22:20:43.263Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1465 | 2024-04-03T22:20:43.263Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1466 | 2024-04-03T22:20:43.283Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1467 | 2024-04-03T22:20:43.283Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1468 | 2024-04-03T22:20:43.303Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1469 | 2024-04-03T22:20:43.303Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1470 | 2024-04-03T22:20:43.322Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1471 | 2024-04-03T22:20:43.322Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1472 | 2024-04-03T22:20:43.342Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1473 | 2024-04-03T22:20:43.342Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1474 | 2024-04-03T22:20:43.361Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1475 | 2024-04-03T22:20:43.361Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1476 | 2024-04-03T22:20:43.380Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1477 | 2024-04-03T22:20:43.380Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1478 | 2024-04-03T22:20:43.400Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1479 | 2024-04-03T22:20:43.401Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1480 | 2024-04-03T22:20:43.420Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1481 | 2024-04-03T22:20:43.420Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1482 | 2024-04-03T22:20:43.440Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1483 | 2024-04-03T22:20:43.440Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1484 | 2024-04-03T22:20:43.460Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1485 | 2024-04-03T22:20:43.460Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1486 | 2024-04-03T22:20:43.480Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1487 | 2024-04-03T22:20:43.480Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1488 | 2024-04-03T22:20:43.499Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1489 | 2024-04-03T22:20:43.499Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1490 | 2024-04-03T22:20:43.518Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1491 | 2024-04-03T22:20:43.518Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1492 | 2024-04-03T22:20:43.539Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1493 | 2024-04-03T22:20:43.539Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1494 | 2024-04-03T22:20:43.559Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1495 | 2024-04-03T22:20:43.559Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1496 | 2024-04-03T22:20:43.560Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1497 | 2024-04-03T22:20:43.560Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1498 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1499 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1500 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1501 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1502 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1503 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1504 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1505 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1506 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1507 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1508 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1509 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1510 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1511 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1512 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1513 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1514 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1515 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1516 | 2024-04-03T22:20:43.658Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1517 | 2024-04-03T22:20:43.658Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 499463
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 4c9eb86c-c76e-4c24-9a73-1cfc0954c65f
response_code = 500
uri = /omicron-zones
|
| 1518 | 2024-04-03T22:20:43.659Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "4c9eb86c-c76e-4c24-9a73-1cfc0954c65f", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:43 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "4c9eb86c-c76e-4c24-9a73-1cfc0954c65f" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 794.316093ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1519 | 2024-04-03T22:20:44.456Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1520 | 2024-04-03T22:20:44.457Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1521 | 2024-04-03T22:20:44.457Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1522 | 2024-04-03T22:20:44.480Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1523 | 2024-04-03T22:20:44.480Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1524 | 2024-04-03T22:20:44.502Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1525 | 2024-04-03T22:20:44.502Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1526 | 2024-04-03T22:20:44.521Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1527 | 2024-04-03T22:20:44.521Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1528 | 2024-04-03T22:20:44.542Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1529 | 2024-04-03T22:20:44.542Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1530 | 2024-04-03T22:20:44.561Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1531 | 2024-04-03T22:20:44.561Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1532 | 2024-04-03T22:20:44.579Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1533 | 2024-04-03T22:20:44.579Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1534 | 2024-04-03T22:20:44.599Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1535 | 2024-04-03T22:20:44.599Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1536 | 2024-04-03T22:20:44.617Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1537 | 2024-04-03T22:20:44.617Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1538 | 2024-04-03T22:20:44.636Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1539 | 2024-04-03T22:20:44.637Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1540 | 2024-04-03T22:20:44.655Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1541 | 2024-04-03T22:20:44.655Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1542 | 2024-04-03T22:20:44.674Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1543 | 2024-04-03T22:20:44.674Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1544 | 2024-04-03T22:20:44.695Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1545 | 2024-04-03T22:20:44.695Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1546 | 2024-04-03T22:20:44.714Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1547 | 2024-04-03T22:20:44.714Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1548 | 2024-04-03T22:20:44.733Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1549 | 2024-04-03T22:20:44.733Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1550 | 2024-04-03T22:20:44.752Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1551 | 2024-04-03T22:20:44.752Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1552 | 2024-04-03T22:20:44.772Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1553 | 2024-04-03T22:20:44.772Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1554 | 2024-04-03T22:20:44.790Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1555 | 2024-04-03T22:20:44.790Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1556 | 2024-04-03T22:20:44.810Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1557 | 2024-04-03T22:20:44.810Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1558 | 2024-04-03T22:20:44.829Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1559 | 2024-04-03T22:20:44.829Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1560 | 2024-04-03T22:20:44.847Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1561 | 2024-04-03T22:20:44.847Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1562 | 2024-04-03T22:20:44.848Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1563 | 2024-04-03T22:20:44.848Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1564 | 2024-04-03T22:20:44.944Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1565 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1566 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1567 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1568 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1569 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1570 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1571 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1572 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1573 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1574 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1575 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1576 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1577 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1578 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1579 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1580 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1581 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1582 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1583 | 2024-04-03T22:20:44.945Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 486989
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = b45cd89e-ecdd-4615-9577-358bf327e09c
response_code = 500
uri = /omicron-zones
|
| 1584 | 2024-04-03T22:20:44.945Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b45cd89e-ecdd-4615-9577-358bf327e09c", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:44 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b45cd89e-ecdd-4615-9577-358bf327e09c" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 830.505925ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1585 | 2024-04-03T22:20:45.779Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1586 | 2024-04-03T22:20:45.779Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1587 | 2024-04-03T22:20:45.779Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1588 | 2024-04-03T22:20:45.803Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1589 | 2024-04-03T22:20:45.803Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1590 | 2024-04-03T22:20:45.823Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1591 | 2024-04-03T22:20:45.823Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1592 | 2024-04-03T22:20:45.842Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1593 | 2024-04-03T22:20:45.842Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1594 | 2024-04-03T22:20:45.861Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1595 | 2024-04-03T22:20:45.861Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1596 | 2024-04-03T22:20:45.879Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1597 | 2024-04-03T22:20:45.879Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1598 | 2024-04-03T22:20:45.900Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1599 | 2024-04-03T22:20:45.900Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1600 | 2024-04-03T22:20:45.919Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1601 | 2024-04-03T22:20:45.919Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1602 | 2024-04-03T22:20:45.939Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1603 | 2024-04-03T22:20:45.939Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1604 | 2024-04-03T22:20:45.958Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1605 | 2024-04-03T22:20:45.958Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1606 | 2024-04-03T22:20:45.979Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1607 | 2024-04-03T22:20:45.979Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1608 | 2024-04-03T22:20:46.000Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1609 | 2024-04-03T22:20:46.000Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1610 | 2024-04-03T22:20:46.020Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1611 | 2024-04-03T22:20:46.020Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1612 | 2024-04-03T22:20:46.039Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1613 | 2024-04-03T22:20:46.039Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1614 | 2024-04-03T22:20:46.059Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1615 | 2024-04-03T22:20:46.059Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1616 | 2024-04-03T22:20:46.077Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1617 | 2024-04-03T22:20:46.077Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1618 | 2024-04-03T22:20:46.096Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1619 | 2024-04-03T22:20:46.096Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1620 | 2024-04-03T22:20:46.115Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1621 | 2024-04-03T22:20:46.115Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1622 | 2024-04-03T22:20:46.133Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1623 | 2024-04-03T22:20:46.133Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1624 | 2024-04-03T22:20:46.152Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1625 | 2024-04-03T22:20:46.152Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1626 | 2024-04-03T22:20:46.170Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1627 | 2024-04-03T22:20:46.170Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1628 | 2024-04-03T22:20:46.170Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1629 | 2024-04-03T22:20:46.170Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1630 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1631 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1632 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1633 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1634 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1635 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1636 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1637 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1638 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1639 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1640 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1641 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1642 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1643 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1644 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1645 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1646 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1647 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1648 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1649 | 2024-04-03T22:20:46.271Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 490584
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 3cab751e-844b-493f-aa81-e4dfec96d41f
response_code = 500
uri = /omicron-zones
|
| 1650 | 2024-04-03T22:20:46.271Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3cab751e-844b-493f-aa81-e4dfec96d41f", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:46 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3cab751e-844b-493f-aa81-e4dfec96d41f" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 877.411854ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1651 | 2024-04-03T22:20:47.152Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1652 | 2024-04-03T22:20:47.153Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1653 | 2024-04-03T22:20:47.153Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1654 | 2024-04-03T22:20:47.176Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1655 | 2024-04-03T22:20:47.176Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1656 | 2024-04-03T22:20:47.197Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1657 | 2024-04-03T22:20:47.197Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1658 | 2024-04-03T22:20:47.218Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1659 | 2024-04-03T22:20:47.218Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1660 | 2024-04-03T22:20:47.237Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1661 | 2024-04-03T22:20:47.237Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1662 | 2024-04-03T22:20:47.256Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1663 | 2024-04-03T22:20:47.256Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1664 | 2024-04-03T22:20:47.276Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1665 | 2024-04-03T22:20:47.277Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1666 | 2024-04-03T22:20:47.297Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1667 | 2024-04-03T22:20:47.297Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1668 | 2024-04-03T22:20:47.317Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1669 | 2024-04-03T22:20:47.317Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1670 | 2024-04-03T22:20:47.338Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1671 | 2024-04-03T22:20:47.339Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1672 | 2024-04-03T22:20:47.359Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1673 | 2024-04-03T22:20:47.359Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1674 | 2024-04-03T22:20:47.379Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1675 | 2024-04-03T22:20:47.379Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1676 | 2024-04-03T22:20:47.398Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1677 | 2024-04-03T22:20:47.398Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1678 | 2024-04-03T22:20:47.418Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1679 | 2024-04-03T22:20:47.418Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1680 | 2024-04-03T22:20:47.437Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1681 | 2024-04-03T22:20:47.437Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1682 | 2024-04-03T22:20:47.458Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1683 | 2024-04-03T22:20:47.458Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1684 | 2024-04-03T22:20:47.478Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1685 | 2024-04-03T22:20:47.478Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1686 | 2024-04-03T22:20:47.496Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1687 | 2024-04-03T22:20:47.496Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1688 | 2024-04-03T22:20:47.517Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1689 | 2024-04-03T22:20:47.517Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1690 | 2024-04-03T22:20:47.540Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1691 | 2024-04-03T22:20:47.540Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1692 | 2024-04-03T22:20:47.562Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1693 | 2024-04-03T22:20:47.562Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1694 | 2024-04-03T22:20:47.562Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1695 | 2024-04-03T22:20:47.562Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1696 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1697 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1698 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1699 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1700 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1701 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1702 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1703 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1704 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1705 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1706 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1707 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1708 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1709 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1710 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1711 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1712 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1713 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1714 | 2024-04-03T22:20:47.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1715 | 2024-04-03T22:20:47.656Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 502384
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 5bc25f58-edc1-4358-a591-01aa714b10ba
response_code = 500
uri = /omicron-zones
|
| 1716 | 2024-04-03T22:20:47.657Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "5bc25f58-edc1-4358-a591-01aa714b10ba", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:47 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "5bc25f58-edc1-4358-a591-01aa714b10ba" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 1.534691598s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1717 | 2024-04-03T22:20:49.197Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1718 | 2024-04-03T22:20:49.198Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1719 | 2024-04-03T22:20:49.198Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1720 | 2024-04-03T22:20:49.221Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1721 | 2024-04-03T22:20:49.221Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1722 | 2024-04-03T22:20:49.245Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1723 | 2024-04-03T22:20:49.245Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1724 | 2024-04-03T22:20:49.264Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1725 | 2024-04-03T22:20:49.264Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1726 | 2024-04-03T22:20:49.282Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1727 | 2024-04-03T22:20:49.282Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1728 | 2024-04-03T22:20:49.302Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1729 | 2024-04-03T22:20:49.302Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1730 | 2024-04-03T22:20:49.322Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1731 | 2024-04-03T22:20:49.322Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1732 | 2024-04-03T22:20:49.340Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1733 | 2024-04-03T22:20:49.340Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1734 | 2024-04-03T22:20:49.359Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1735 | 2024-04-03T22:20:49.359Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1736 | 2024-04-03T22:20:49.378Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1737 | 2024-04-03T22:20:49.379Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1738 | 2024-04-03T22:20:49.398Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1739 | 2024-04-03T22:20:49.398Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1740 | 2024-04-03T22:20:49.417Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1741 | 2024-04-03T22:20:49.417Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1742 | 2024-04-03T22:20:49.437Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1743 | 2024-04-03T22:20:49.437Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1744 | 2024-04-03T22:20:49.456Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1745 | 2024-04-03T22:20:49.456Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1746 | 2024-04-03T22:20:49.476Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1747 | 2024-04-03T22:20:49.476Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1748 | 2024-04-03T22:20:49.497Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1749 | 2024-04-03T22:20:49.497Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1750 | 2024-04-03T22:20:49.517Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1751 | 2024-04-03T22:20:49.517Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1752 | 2024-04-03T22:20:49.538Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1753 | 2024-04-03T22:20:49.538Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1754 | 2024-04-03T22:20:49.558Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1755 | 2024-04-03T22:20:49.558Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1756 | 2024-04-03T22:20:49.577Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1757 | 2024-04-03T22:20:49.577Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1758 | 2024-04-03T22:20:49.599Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1759 | 2024-04-03T22:20:49.599Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1760 | 2024-04-03T22:20:49.599Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1761 | 2024-04-03T22:20:49.599Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1762 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1763 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1764 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1765 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1766 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1767 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1768 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1769 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1770 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1771 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1772 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1773 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1774 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1775 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1776 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1777 | 2024-04-03T22:20:49.695Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1778 | 2024-04-03T22:20:49.696Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1779 | 2024-04-03T22:20:49.696Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1780 | 2024-04-03T22:20:49.696Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1781 | 2024-04-03T22:20:49.696Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 496564
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = cc8d4c0b-8454-428d-a2cc-0e8cde1c6481
response_code = 500
uri = /omicron-zones
|
| 1782 | 2024-04-03T22:20:49.696Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "cc8d4c0b-8454-428d-a2cc-0e8cde1c6481", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:49 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "cc8d4c0b-8454-428d-a2cc-0e8cde1c6481" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 682.573966ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1783 | 2024-04-03T22:20:50.381Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1784 | 2024-04-03T22:20:50.381Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1785 | 2024-04-03T22:20:50.381Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1786 | 2024-04-03T22:20:50.404Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1787 | 2024-04-03T22:20:50.404Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1788 | 2024-04-03T22:20:50.423Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1789 | 2024-04-03T22:20:50.423Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1790 | 2024-04-03T22:20:50.444Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1791 | 2024-04-03T22:20:50.444Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1792 | 2024-04-03T22:20:50.463Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1793 | 2024-04-03T22:20:50.463Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1794 | 2024-04-03T22:20:50.483Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1795 | 2024-04-03T22:20:50.483Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1796 | 2024-04-03T22:20:50.502Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1797 | 2024-04-03T22:20:50.503Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1798 | 2024-04-03T22:20:50.521Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1799 | 2024-04-03T22:20:50.522Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1800 | 2024-04-03T22:20:50.541Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1801 | 2024-04-03T22:20:50.541Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1802 | 2024-04-03T22:20:50.562Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1803 | 2024-04-03T22:20:50.562Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1804 | 2024-04-03T22:20:50.581Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1805 | 2024-04-03T22:20:50.581Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1806 | 2024-04-03T22:20:50.600Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1807 | 2024-04-03T22:20:50.600Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1808 | 2024-04-03T22:20:50.620Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1809 | 2024-04-03T22:20:50.620Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1810 | 2024-04-03T22:20:50.639Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1811 | 2024-04-03T22:20:50.639Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1812 | 2024-04-03T22:20:50.658Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1813 | 2024-04-03T22:20:50.658Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1814 | 2024-04-03T22:20:50.677Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1815 | 2024-04-03T22:20:50.677Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1816 | 2024-04-03T22:20:50.696Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1817 | 2024-04-03T22:20:50.696Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1818 | 2024-04-03T22:20:50.715Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1819 | 2024-04-03T22:20:50.715Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1820 | 2024-04-03T22:20:50.735Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1821 | 2024-04-03T22:20:50.735Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1822 | 2024-04-03T22:20:50.754Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1823 | 2024-04-03T22:20:50.754Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1824 | 2024-04-03T22:20:50.774Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1825 | 2024-04-03T22:20:50.774Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1826 | 2024-04-03T22:20:50.774Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1827 | 2024-04-03T22:20:50.774Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1828 | 2024-04-03T22:20:50.871Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1829 | 2024-04-03T22:20:50.871Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1830 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1831 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1832 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1833 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1834 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1835 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1836 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1837 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1838 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1839 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1840 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1841 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1842 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1843 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1844 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1845 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1846 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1847 | 2024-04-03T22:20:50.872Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 489187
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = b7d750d4-e1ff-4142-8fc0-8baa83ad5ec3
response_code = 500
uri = /omicron-zones
|
| 1848 | 2024-04-03T22:20:50.872Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b7d750d4-e1ff-4142-8fc0-8baa83ad5ec3", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:50 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b7d750d4-e1ff-4142-8fc0-8baa83ad5ec3" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 2.214735206s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1849 | 2024-04-03T22:20:53.094Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1850 | 2024-04-03T22:20:53.094Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1851 | 2024-04-03T22:20:53.094Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1852 | 2024-04-03T22:20:53.116Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1853 | 2024-04-03T22:20:53.116Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1854 | 2024-04-03T22:20:53.137Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1855 | 2024-04-03T22:20:53.137Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1856 | 2024-04-03T22:20:53.156Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1857 | 2024-04-03T22:20:53.156Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1858 | 2024-04-03T22:20:53.175Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1859 | 2024-04-03T22:20:53.175Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1860 | 2024-04-03T22:20:53.194Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1861 | 2024-04-03T22:20:53.194Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1862 | 2024-04-03T22:20:53.213Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1863 | 2024-04-03T22:20:53.213Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1864 | 2024-04-03T22:20:53.232Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1865 | 2024-04-03T22:20:53.232Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1866 | 2024-04-03T22:20:53.252Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1867 | 2024-04-03T22:20:53.252Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1868 | 2024-04-03T22:20:53.273Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1869 | 2024-04-03T22:20:53.273Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1870 | 2024-04-03T22:20:53.294Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1871 | 2024-04-03T22:20:53.294Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1872 | 2024-04-03T22:20:53.314Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1873 | 2024-04-03T22:20:53.314Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1874 | 2024-04-03T22:20:53.333Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1875 | 2024-04-03T22:20:53.333Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1876 | 2024-04-03T22:20:53.354Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1877 | 2024-04-03T22:20:53.354Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1878 | 2024-04-03T22:20:53.374Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1879 | 2024-04-03T22:20:53.374Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1880 | 2024-04-03T22:20:53.393Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1881 | 2024-04-03T22:20:53.393Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1882 | 2024-04-03T22:20:53.412Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1883 | 2024-04-03T22:20:53.412Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1884 | 2024-04-03T22:20:53.431Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1885 | 2024-04-03T22:20:53.431Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1886 | 2024-04-03T22:20:53.453Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1887 | 2024-04-03T22:20:53.453Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1888 | 2024-04-03T22:20:53.471Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1889 | 2024-04-03T22:20:53.471Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1890 | 2024-04-03T22:20:53.492Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1891 | 2024-04-03T22:20:53.492Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1892 | 2024-04-03T22:20:53.492Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1893 | 2024-04-03T22:20:53.492Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1894 | 2024-04-03T22:20:53.585Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1895 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1896 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1897 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1898 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1899 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1900 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1901 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1902 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1903 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1904 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1905 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1906 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1907 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1908 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1909 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1910 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1911 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1912 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1913 | 2024-04-03T22:20:53.586Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 490498
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = f1b68c57-5cb9-4c21-8f22-bb95c1397dc0
response_code = 500
uri = /omicron-zones
|
| 1914 | 2024-04-03T22:20:53.586Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f1b68c57-5cb9-4c21-8f22-bb95c1397dc0", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:53 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f1b68c57-5cb9-4c21-8f22-bb95c1397dc0" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 2.054333757s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1915 | 2024-04-03T22:20:55.647Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1916 | 2024-04-03T22:20:55.648Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1917 | 2024-04-03T22:20:55.648Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1918 | 2024-04-03T22:20:55.671Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1919 | 2024-04-03T22:20:55.671Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1920 | 2024-04-03T22:20:55.692Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1921 | 2024-04-03T22:20:55.692Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1922 | 2024-04-03T22:20:55.713Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1923 | 2024-04-03T22:20:55.713Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1924 | 2024-04-03T22:20:55.733Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1925 | 2024-04-03T22:20:55.733Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1926 | 2024-04-03T22:20:55.753Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1927 | 2024-04-03T22:20:55.753Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1928 | 2024-04-03T22:20:55.771Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1929 | 2024-04-03T22:20:55.772Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1930 | 2024-04-03T22:20:55.790Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1931 | 2024-04-03T22:20:55.790Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1932 | 2024-04-03T22:20:55.813Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1933 | 2024-04-03T22:20:55.813Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1934 | 2024-04-03T22:20:55.832Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1935 | 2024-04-03T22:20:55.832Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 1936 | 2024-04-03T22:20:55.851Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1937 | 2024-04-03T22:20:55.851Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 1938 | 2024-04-03T22:20:55.871Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1939 | 2024-04-03T22:20:55.871Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 1940 | 2024-04-03T22:20:55.892Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1941 | 2024-04-03T22:20:55.893Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 1942 | 2024-04-03T22:20:55.912Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1943 | 2024-04-03T22:20:55.912Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 1944 | 2024-04-03T22:20:55.930Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1945 | 2024-04-03T22:20:55.930Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 1946 | 2024-04-03T22:20:55.948Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1947 | 2024-04-03T22:20:55.948Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 1948 | 2024-04-03T22:20:55.968Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1949 | 2024-04-03T22:20:55.968Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 1950 | 2024-04-03T22:20:55.989Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1951 | 2024-04-03T22:20:55.989Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 1952 | 2024-04-03T22:20:56.011Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1953 | 2024-04-03T22:20:56.011Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 1954 | 2024-04-03T22:20:56.030Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1955 | 2024-04-03T22:20:56.030Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 1956 | 2024-04-03T22:20:56.050Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1957 | 2024-04-03T22:20:56.050Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1958 | 2024-04-03T22:20:56.051Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1959 | 2024-04-03T22:20:56.051Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1960 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 1961 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 1962 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 1963 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 1964 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 1965 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 1966 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 1967 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 1968 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 1969 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 1970 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 1971 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 1972 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 1973 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 1974 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 1975 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 1976 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 1977 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 1978 | 2024-04-03T22:20:56.145Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 1979 | 2024-04-03T22:20:56.145Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 496458
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 1f184525-117d-4cfd-9ca9-c1075311e5b8
response_code = 500
uri = /omicron-zones
|
| 1980 | 2024-04-03T22:20:56.146Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "1f184525-117d-4cfd-9ca9-c1075311e5b8", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:55 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "1f184525-117d-4cfd-9ca9-c1075311e5b8" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 2.876012879s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1981 | 2024-04-03T22:20:59.031Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1982 | 2024-04-03T22:20:59.031Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1983 | 2024-04-03T22:20:59.031Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 1984 | 2024-04-03T22:20:59.055Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1985 | 2024-04-03T22:20:59.055Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 1986 | 2024-04-03T22:20:59.076Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1987 | 2024-04-03T22:20:59.076Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 1988 | 2024-04-03T22:20:59.095Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1989 | 2024-04-03T22:20:59.095Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1990 | 2024-04-03T22:20:59.114Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1991 | 2024-04-03T22:20:59.114Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1992 | 2024-04-03T22:20:59.135Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1993 | 2024-04-03T22:20:59.135Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1994 | 2024-04-03T22:20:59.155Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1995 | 2024-04-03T22:20:59.155Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1996 | 2024-04-03T22:20:59.175Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1997 | 2024-04-03T22:20:59.175Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 1998 | 2024-04-03T22:20:59.197Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1999 | 2024-04-03T22:20:59.197Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2000 | 2024-04-03T22:20:59.218Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2001 | 2024-04-03T22:20:59.218Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2002 | 2024-04-03T22:20:59.239Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2003 | 2024-04-03T22:20:59.239Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 2004 | 2024-04-03T22:20:59.259Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2005 | 2024-04-03T22:20:59.259Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 2006 | 2024-04-03T22:20:59.278Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2007 | 2024-04-03T22:20:59.278Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 2008 | 2024-04-03T22:20:59.298Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2009 | 2024-04-03T22:20:59.298Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 2010 | 2024-04-03T22:20:59.317Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2011 | 2024-04-03T22:20:59.317Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 2012 | 2024-04-03T22:20:59.337Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2013 | 2024-04-03T22:20:59.337Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 2014 | 2024-04-03T22:20:59.358Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2015 | 2024-04-03T22:20:59.358Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 2016 | 2024-04-03T22:20:59.377Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2017 | 2024-04-03T22:20:59.377Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 2018 | 2024-04-03T22:20:59.397Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2019 | 2024-04-03T22:20:59.397Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 2020 | 2024-04-03T22:20:59.418Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2021 | 2024-04-03T22:20:59.420Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 2022 | 2024-04-03T22:20:59.437Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2023 | 2024-04-03T22:20:59.437Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2024 | 2024-04-03T22:20:59.437Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2025 | 2024-04-03T22:20:59.437Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2026 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 2027 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2028 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2029 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2030 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2031 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2032 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2033 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2034 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2035 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2036 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2037 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2038 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2039 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2040 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2041 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2042 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2043 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2044 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2045 | 2024-04-03T22:20:59.533Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 500848
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 4fe31a22-1798-45d9-9f40-fc3fa6305f02
response_code = 500
uri = /omicron-zones
|
| 2046 | 2024-04-03T22:20:59.533Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "4fe31a22-1798-45d9-9f40-fc3fa6305f02", "content-length": "124", "date": "Wed, 03 Apr 2024 22:20:59 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "4fe31a22-1798-45d9-9f40-fc3fa6305f02" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 3.617705798s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2047 | 2024-04-03T22:21:03.162Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2048 | 2024-04-03T22:21:03.163Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2049 | 2024-04-03T22:21:03.163Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 2050 | 2024-04-03T22:21:03.186Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2051 | 2024-04-03T22:21:03.186Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 2052 | 2024-04-03T22:21:03.208Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2053 | 2024-04-03T22:21:03.208Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 2054 | 2024-04-03T22:21:03.228Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2055 | 2024-04-03T22:21:03.228Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2056 | 2024-04-03T22:21:03.249Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2057 | 2024-04-03T22:21:03.249Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2058 | 2024-04-03T22:21:03.270Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2059 | 2024-04-03T22:21:03.270Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2060 | 2024-04-03T22:21:03.291Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2061 | 2024-04-03T22:21:03.291Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2062 | 2024-04-03T22:21:03.310Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2063 | 2024-04-03T22:21:03.310Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2064 | 2024-04-03T22:21:03.329Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2065 | 2024-04-03T22:21:03.329Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2066 | 2024-04-03T22:21:03.350Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2067 | 2024-04-03T22:21:03.350Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2068 | 2024-04-03T22:21:03.370Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2069 | 2024-04-03T22:21:03.370Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 2070 | 2024-04-03T22:21:03.390Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2071 | 2024-04-03T22:21:03.390Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 2072 | 2024-04-03T22:21:03.409Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2073 | 2024-04-03T22:21:03.409Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 2074 | 2024-04-03T22:21:03.429Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2075 | 2024-04-03T22:21:03.429Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 2076 | 2024-04-03T22:21:03.450Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2077 | 2024-04-03T22:21:03.450Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 2078 | 2024-04-03T22:21:03.472Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2079 | 2024-04-03T22:21:03.472Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 2080 | 2024-04-03T22:21:03.492Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2081 | 2024-04-03T22:21:03.492Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 2082 | 2024-04-03T22:21:03.514Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2083 | 2024-04-03T22:21:03.514Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 2084 | 2024-04-03T22:21:03.535Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2085 | 2024-04-03T22:21:03.535Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 2086 | 2024-04-03T22:21:03.556Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2087 | 2024-04-03T22:21:03.556Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 2088 | 2024-04-03T22:21:03.577Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2089 | 2024-04-03T22:21:03.577Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2090 | 2024-04-03T22:21:03.577Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2091 | 2024-04-03T22:21:03.577Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2092 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2093 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2094 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2095 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2096 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2097 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2098 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2099 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2100 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2101 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2102 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2103 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2104 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2105 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2106 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2107 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2108 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2109 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2110 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 2111 | 2024-04-03T22:21:03.677Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 512877
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 06fd1c38-d301-4f8a-97b7-a87ab3ae8208
response_code = 500
uri = /omicron-zones
|
| 2112 | 2024-04-03T22:21:03.677Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "06fd1c38-d301-4f8a-97b7-a87ab3ae8208", "content-length": "124", "date": "Wed, 03 Apr 2024 22:21:03 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "06fd1c38-d301-4f8a-97b7-a87ab3ae8208" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 4.726115793s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2113 | 2024-04-03T22:21:08.417Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2114 | 2024-04-03T22:21:08.417Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2115 | 2024-04-03T22:21:08.417Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 2116 | 2024-04-03T22:21:08.440Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2117 | 2024-04-03T22:21:08.440Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 2118 | 2024-04-03T22:21:08.461Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2119 | 2024-04-03T22:21:08.461Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 2120 | 2024-04-03T22:21:08.481Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2121 | 2024-04-03T22:21:08.483Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2122 | 2024-04-03T22:21:08.499Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2123 | 2024-04-03T22:21:08.499Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2124 | 2024-04-03T22:21:08.518Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2125 | 2024-04-03T22:21:08.518Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2126 | 2024-04-03T22:21:08.536Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2127 | 2024-04-03T22:21:08.536Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2128 | 2024-04-03T22:21:08.555Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2129 | 2024-04-03T22:21:08.555Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2130 | 2024-04-03T22:21:08.574Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2131 | 2024-04-03T22:21:08.574Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2132 | 2024-04-03T22:21:08.592Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2133 | 2024-04-03T22:21:08.592Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2134 | 2024-04-03T22:21:08.611Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2135 | 2024-04-03T22:21:08.611Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 2136 | 2024-04-03T22:21:08.629Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2137 | 2024-04-03T22:21:08.630Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 2138 | 2024-04-03T22:21:08.648Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2139 | 2024-04-03T22:21:08.648Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 2140 | 2024-04-03T22:21:08.667Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2141 | 2024-04-03T22:21:08.667Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 2142 | 2024-04-03T22:21:08.685Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2143 | 2024-04-03T22:21:08.685Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 2144 | 2024-04-03T22:21:08.703Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2145 | 2024-04-03T22:21:08.703Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 2146 | 2024-04-03T22:21:08.721Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2147 | 2024-04-03T22:21:08.721Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 2148 | 2024-04-03T22:21:08.739Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2149 | 2024-04-03T22:21:08.739Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 2150 | 2024-04-03T22:21:08.757Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2151 | 2024-04-03T22:21:08.757Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 2152 | 2024-04-03T22:21:08.777Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2153 | 2024-04-03T22:21:08.779Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 2154 | 2024-04-03T22:21:08.795Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2155 | 2024-04-03T22:21:08.795Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2156 | 2024-04-03T22:21:08.795Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2157 | 2024-04-03T22:21:08.795Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2158 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2159 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2160 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 2161 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2162 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2163 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2164 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2165 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2166 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2167 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2168 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2169 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2170 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2171 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2172 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2173 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2174 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2175 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2176 | 2024-04-03T22:21:08.887Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2177 | 2024-04-03T22:21:08.887Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 469149
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = daed0e8f-89ee-40d8-8a49-9a77acdb7438
response_code = 500
uri = /omicron-zones
|
| 2178 | 2024-04-03T22:21:08.888Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "daed0e8f-89ee-40d8-8a49-9a77acdb7438", "content-length": "124", "date": "Wed, 03 Apr 2024 22:21:08 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "daed0e8f-89ee-40d8-8a49-9a77acdb7438" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 5.580989s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2179 | 2024-04-03T22:21:09.935Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:52308
|
| 2180 | 2024-04-03T22:21:09.935Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 52
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:52308
req_id = 43e26eb9-fbe0-4fe8-849b-2c0656f368d0
response_code = 200
uri = /baseboard
|
| 2181 | 2024-04-03T22:21:14.486Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2182 | 2024-04-03T22:21:14.486Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2183 | 2024-04-03T22:21:14.486Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 2184 | 2024-04-03T22:21:14.509Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2185 | 2024-04-03T22:21:14.509Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 2186 | 2024-04-03T22:21:14.531Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2187 | 2024-04-03T22:21:14.531Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 2188 | 2024-04-03T22:21:14.550Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2189 | 2024-04-03T22:21:14.550Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2190 | 2024-04-03T22:21:14.570Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2191 | 2024-04-03T22:21:14.572Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2192 | 2024-04-03T22:21:14.590Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2193 | 2024-04-03T22:21:14.590Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2194 | 2024-04-03T22:21:14.609Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2195 | 2024-04-03T22:21:14.609Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2196 | 2024-04-03T22:21:14.632Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2197 | 2024-04-03T22:21:14.632Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2198 | 2024-04-03T22:21:14.653Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2199 | 2024-04-03T22:21:14.654Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2200 | 2024-04-03T22:21:14.675Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2201 | 2024-04-03T22:21:14.675Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2202 | 2024-04-03T22:21:14.695Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2203 | 2024-04-03T22:21:14.695Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 2204 | 2024-04-03T22:21:14.714Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2205 | 2024-04-03T22:21:14.714Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 2206 | 2024-04-03T22:21:14.732Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2207 | 2024-04-03T22:21:14.732Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 2208 | 2024-04-03T22:21:14.751Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2209 | 2024-04-03T22:21:14.751Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 2210 | 2024-04-03T22:21:14.769Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2211 | 2024-04-03T22:21:14.769Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 2212 | 2024-04-03T22:21:14.788Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2213 | 2024-04-03T22:21:14.788Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 2214 | 2024-04-03T22:21:14.807Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2215 | 2024-04-03T22:21:14.807Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 2216 | 2024-04-03T22:21:14.826Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2217 | 2024-04-03T22:21:14.826Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 2218 | 2024-04-03T22:21:14.845Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2219 | 2024-04-03T22:21:14.845Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 2220 | 2024-04-03T22:21:14.863Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2221 | 2024-04-03T22:21:14.863Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 2222 | 2024-04-03T22:21:14.881Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2223 | 2024-04-03T22:21:14.881Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2224 | 2024-04-03T22:21:14.881Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2225 | 2024-04-03T22:21:14.881Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2226 | 2024-04-03T22:21:14.978Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2227 | 2024-04-03T22:21:14.978Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2228 | 2024-04-03T22:21:14.978Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2229 | 2024-04-03T22:21:14.978Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2230 | 2024-04-03T22:21:14.978Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2231 | 2024-04-03T22:21:14.978Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2232 | 2024-04-03T22:21:14.978Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2233 | 2024-04-03T22:21:14.978Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2234 | 2024-04-03T22:21:14.978Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2235 | 2024-04-03T22:21:14.978Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2236 | 2024-04-03T22:21:14.978Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2237 | 2024-04-03T22:21:14.979Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2238 | 2024-04-03T22:21:14.979Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2239 | 2024-04-03T22:21:14.979Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2240 | 2024-04-03T22:21:14.979Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2241 | 2024-04-03T22:21:14.979Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2242 | 2024-04-03T22:21:14.979Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 2243 | 2024-04-03T22:21:14.979Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2244 | 2024-04-03T22:21:14.979Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2245 | 2024-04-03T22:21:14.979Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 491450
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 1bbc7b1b-2661-4c25-af9a-adf9dab1cabb
response_code = 500
uri = /omicron-zones
|
| 2246 | 2024-04-03T22:21:14.979Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "1bbc7b1b-2661-4c25-af9a-adf9dab1cabb", "content-length": "124", "date": "Wed, 03 Apr 2024 22:21:14 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "1bbc7b1b-2661-4c25-af9a-adf9dab1cabb" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 5.135476024s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2247 | 2024-04-03T22:21:20.129Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2248 | 2024-04-03T22:21:20.130Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2249 | 2024-04-03T22:21:20.130Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 2250 | 2024-04-03T22:21:20.153Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2251 | 2024-04-03T22:21:20.153Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 2252 | 2024-04-03T22:21:20.174Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2253 | 2024-04-03T22:21:20.174Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 2254 | 2024-04-03T22:21:20.193Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2255 | 2024-04-03T22:21:20.193Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2256 | 2024-04-03T22:21:20.212Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2257 | 2024-04-03T22:21:20.212Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2258 | 2024-04-03T22:21:20.231Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2259 | 2024-04-03T22:21:20.231Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2260 | 2024-04-03T22:21:20.249Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2261 | 2024-04-03T22:21:20.250Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2262 | 2024-04-03T22:21:20.268Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2263 | 2024-04-03T22:21:20.268Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2264 | 2024-04-03T22:21:20.287Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2265 | 2024-04-03T22:21:20.287Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2266 | 2024-04-03T22:21:20.306Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2267 | 2024-04-03T22:21:20.306Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2268 | 2024-04-03T22:21:20.326Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2269 | 2024-04-03T22:21:20.326Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 2270 | 2024-04-03T22:21:20.345Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2271 | 2024-04-03T22:21:20.345Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 2272 | 2024-04-03T22:21:20.364Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2273 | 2024-04-03T22:21:20.364Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 2274 | 2024-04-03T22:21:20.383Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2275 | 2024-04-03T22:21:20.383Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 2276 | 2024-04-03T22:21:20.402Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2277 | 2024-04-03T22:21:20.402Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 2278 | 2024-04-03T22:21:20.421Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2279 | 2024-04-03T22:21:20.421Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 2280 | 2024-04-03T22:21:20.440Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2281 | 2024-04-03T22:21:20.440Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 2282 | 2024-04-03T22:21:20.459Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2283 | 2024-04-03T22:21:20.459Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 2284 | 2024-04-03T22:21:20.478Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2285 | 2024-04-03T22:21:20.478Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 2286 | 2024-04-03T22:21:20.497Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2287 | 2024-04-03T22:21:20.497Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 2288 | 2024-04-03T22:21:20.515Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2289 | 2024-04-03T22:21:20.515Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2290 | 2024-04-03T22:21:20.515Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2291 | 2024-04-03T22:21:20.515Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2292 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2293 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2294 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2295 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2296 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2297 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2298 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2299 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2300 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2301 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2302 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2303 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 2304 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2305 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2306 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2307 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2308 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2309 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2310 | 2024-04-03T22:21:20.609Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2311 | 2024-04-03T22:21:20.610Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 478603
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 3eab83d5-8047-42aa-b2c4-174a84143f03
response_code = 500
uri = /omicron-zones
|
| 2312 | 2024-04-03T22:21:20.610Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3eab83d5-8047-42aa-b2c4-174a84143f03", "content-length": "124", "date": "Wed, 03 Apr 2024 22:21:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3eab83d5-8047-42aa-b2c4-174a84143f03" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 3.489561213s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2313 | 2024-04-03T22:21:24.109Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2314 | 2024-04-03T22:21:24.110Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2315 | 2024-04-03T22:21:24.110Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 2316 | 2024-04-03T22:21:24.132Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2317 | 2024-04-03T22:21:24.132Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 2318 | 2024-04-03T22:21:24.153Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2319 | 2024-04-03T22:21:24.153Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 2320 | 2024-04-03T22:21:24.172Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2321 | 2024-04-03T22:21:24.172Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2322 | 2024-04-03T22:21:24.191Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2323 | 2024-04-03T22:21:24.191Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2324 | 2024-04-03T22:21:24.208Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2325 | 2024-04-03T22:21:24.208Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2326 | 2024-04-03T22:21:24.226Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2327 | 2024-04-03T22:21:24.226Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2328 | 2024-04-03T22:21:24.244Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2329 | 2024-04-03T22:21:24.244Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2330 | 2024-04-03T22:21:24.262Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2331 | 2024-04-03T22:21:24.262Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2332 | 2024-04-03T22:21:24.280Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2333 | 2024-04-03T22:21:24.280Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2334 | 2024-04-03T22:21:24.299Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2335 | 2024-04-03T22:21:24.299Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 2336 | 2024-04-03T22:21:24.318Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2337 | 2024-04-03T22:21:24.318Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 2338 | 2024-04-03T22:21:24.337Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2339 | 2024-04-03T22:21:24.337Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 2340 | 2024-04-03T22:21:24.356Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2341 | 2024-04-03T22:21:24.356Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 2342 | 2024-04-03T22:21:24.375Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2343 | 2024-04-03T22:21:24.375Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 2344 | 2024-04-03T22:21:24.395Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2345 | 2024-04-03T22:21:24.395Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 2346 | 2024-04-03T22:21:24.414Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2347 | 2024-04-03T22:21:24.414Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 2348 | 2024-04-03T22:21:24.433Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2349 | 2024-04-03T22:21:24.433Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 2350 | 2024-04-03T22:21:24.454Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2351 | 2024-04-03T22:21:24.454Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 2352 | 2024-04-03T22:21:24.473Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2353 | 2024-04-03T22:21:24.473Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 2354 | 2024-04-03T22:21:24.491Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2355 | 2024-04-03T22:21:24.491Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2356 | 2024-04-03T22:21:24.492Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2357 | 2024-04-03T22:21:24.492Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2358 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2359 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2360 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 2361 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2362 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2363 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2364 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2365 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2366 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2367 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2368 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2369 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2370 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2371 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2372 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2373 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2374 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2375 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2376 | 2024-04-03T22:21:24.586Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2377 | 2024-04-03T22:21:24.586Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 475471
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = c89edc9f-9bd0-414a-9514-77b29cc8f5af
response_code = 500
uri = /omicron-zones
|
| 2378 | 2024-04-03T22:21:24.587Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "c89edc9f-9bd0-414a-9514-77b29cc8f5af", "content-length": "124", "date": "Wed, 03 Apr 2024 22:21:24 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "c89edc9f-9bd0-414a-9514-77b29cc8f5af" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 4.087787662s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2379 | 2024-04-03T22:21:28.686Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2380 | 2024-04-03T22:21:28.687Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2381 | 2024-04-03T22:21:28.687Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 2382 | 2024-04-03T22:21:28.709Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2383 | 2024-04-03T22:21:28.709Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 2384 | 2024-04-03T22:21:28.731Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2385 | 2024-04-03T22:21:28.731Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 2386 | 2024-04-03T22:21:28.751Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2387 | 2024-04-03T22:21:28.751Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2388 | 2024-04-03T22:21:28.769Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2389 | 2024-04-03T22:21:28.769Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2390 | 2024-04-03T22:21:28.785Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2391 | 2024-04-03T22:21:28.785Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2392 | 2024-04-03T22:21:28.802Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2393 | 2024-04-03T22:21:28.802Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2394 | 2024-04-03T22:21:28.821Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2395 | 2024-04-03T22:21:28.821Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2396 | 2024-04-03T22:21:28.839Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2397 | 2024-04-03T22:21:28.839Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2398 | 2024-04-03T22:21:28.858Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2399 | 2024-04-03T22:21:28.858Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2400 | 2024-04-03T22:21:28.876Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2401 | 2024-04-03T22:21:28.876Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 2402 | 2024-04-03T22:21:28.894Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2403 | 2024-04-03T22:21:28.894Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 2404 | 2024-04-03T22:21:28.912Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2405 | 2024-04-03T22:21:28.912Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 2406 | 2024-04-03T22:21:28.930Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2407 | 2024-04-03T22:21:28.930Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 2408 | 2024-04-03T22:21:28.948Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2409 | 2024-04-03T22:21:28.948Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 2410 | 2024-04-03T22:21:28.967Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2411 | 2024-04-03T22:21:28.967Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 2412 | 2024-04-03T22:21:28.985Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2413 | 2024-04-03T22:21:28.985Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 2414 | 2024-04-03T22:21:29.003Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2415 | 2024-04-03T22:21:29.004Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 2416 | 2024-04-03T22:21:29.024Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2417 | 2024-04-03T22:21:29.024Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 2418 | 2024-04-03T22:21:29.043Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2419 | 2024-04-03T22:21:29.043Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 2420 | 2024-04-03T22:21:29.061Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2421 | 2024-04-03T22:21:29.061Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2422 | 2024-04-03T22:21:29.061Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2423 | 2024-04-03T22:21:29.061Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2424 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 2425 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2426 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2427 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2428 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2429 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2430 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2431 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2432 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2433 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2434 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2435 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2436 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2437 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2438 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2439 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2440 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2441 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2442 | 2024-04-03T22:21:29.155Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2443 | 2024-04-03T22:21:29.155Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 467262
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 77e261c1-7405-41b6-8673-185d1f09eadd
response_code = 500
uri = /omicron-zones
|
| 2444 | 2024-04-03T22:21:29.156Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "77e261c1-7405-41b6-8673-185d1f09eadd", "content-length": "124", "date": "Wed, 03 Apr 2024 22:21:28 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "77e261c1-7405-41b6-8673-185d1f09eadd" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 5.930776568s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2445 | 2024-04-03T22:21:35.103Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2446 | 2024-04-03T22:21:35.104Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2447 | 2024-04-03T22:21:35.104Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 2448 | 2024-04-03T22:21:35.130Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2449 | 2024-04-03T22:21:35.130Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 2450 | 2024-04-03T22:21:35.154Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2451 | 2024-04-03T22:21:35.154Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 2452 | 2024-04-03T22:21:35.177Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2453 | 2024-04-03T22:21:35.177Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2454 | 2024-04-03T22:21:35.199Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2455 | 2024-04-03T22:21:35.199Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2456 | 2024-04-03T22:21:35.223Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2457 | 2024-04-03T22:21:35.223Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2458 | 2024-04-03T22:21:35.245Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2459 | 2024-04-03T22:21:35.245Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2460 | 2024-04-03T22:21:35.269Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2461 | 2024-04-03T22:21:35.269Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2462 | 2024-04-03T22:21:35.290Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2463 | 2024-04-03T22:21:35.290Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2464 | 2024-04-03T22:21:35.313Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2465 | 2024-04-03T22:21:35.313Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2466 | 2024-04-03T22:21:35.339Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2467 | 2024-04-03T22:21:35.339Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 2468 | 2024-04-03T22:21:35.364Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2469 | 2024-04-03T22:21:35.364Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 2470 | 2024-04-03T22:21:35.388Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2471 | 2024-04-03T22:21:35.388Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 2472 | 2024-04-03T22:21:35.413Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2473 | 2024-04-03T22:21:35.413Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 2474 | 2024-04-03T22:21:35.438Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2475 | 2024-04-03T22:21:35.439Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 2476 | 2024-04-03T22:21:35.463Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2477 | 2024-04-03T22:21:35.463Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 2478 | 2024-04-03T22:21:35.488Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2479 | 2024-04-03T22:21:35.488Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 2480 | 2024-04-03T22:21:35.511Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2481 | 2024-04-03T22:21:35.511Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 2482 | 2024-04-03T22:21:35.536Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2483 | 2024-04-03T22:21:35.536Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 2484 | 2024-04-03T22:21:35.561Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2485 | 2024-04-03T22:21:35.561Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 2486 | 2024-04-03T22:21:35.587Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2487 | 2024-04-03T22:21:35.587Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2488 | 2024-04-03T22:21:35.587Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2489 | 2024-04-03T22:21:35.587Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2490 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 2491 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2492 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2493 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2494 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2495 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2496 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2497 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2498 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2499 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2500 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2501 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2502 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2503 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2504 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2505 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2506 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2507 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2508 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2509 | 2024-04-03T22:21:35.699Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 594362
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 26dddc0d-4e03-47c9-8f8e-27b899d0a992
response_code = 500
uri = /omicron-zones
|
| 2510 | 2024-04-03T22:21:35.699Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "26dddc0d-4e03-47c9-8f8e-27b899d0a992", "content-length": "124", "date": "Wed, 03 Apr 2024 22:21:35 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "26dddc0d-4e03-47c9-8f8e-27b899d0a992" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 4.778500562s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2511 | 2024-04-03T22:21:40.047Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:64745
|
| 2512 | 2024-04-03T22:21:40.047Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 49
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:64745
req_id = c05180f3-6cde-478f-81a5-380d5d2cfcfc
response_code = 200
uri = /baseboard
|
| 2513 | 2024-04-03T22:21:40.491Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2514 | 2024-04-03T22:21:40.491Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2515 | 2024-04-03T22:21:40.491Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 2516 | 2024-04-03T22:21:40.513Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2517 | 2024-04-03T22:21:40.513Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 2518 | 2024-04-03T22:21:40.531Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2519 | 2024-04-03T22:21:40.531Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 2520 | 2024-04-03T22:21:40.551Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2521 | 2024-04-03T22:21:40.551Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2522 | 2024-04-03T22:21:40.569Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2523 | 2024-04-03T22:21:40.569Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2524 | 2024-04-03T22:21:40.587Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2525 | 2024-04-03T22:21:40.587Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2526 | 2024-04-03T22:21:40.605Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2527 | 2024-04-03T22:21:40.605Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2528 | 2024-04-03T22:21:40.623Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2529 | 2024-04-03T22:21:40.623Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2530 | 2024-04-03T22:21:40.640Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2531 | 2024-04-03T22:21:40.641Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2532 | 2024-04-03T22:21:40.658Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2533 | 2024-04-03T22:21:40.658Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2534 | 2024-04-03T22:21:40.677Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2535 | 2024-04-03T22:21:40.677Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 2536 | 2024-04-03T22:21:40.696Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2537 | 2024-04-03T22:21:40.696Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 2538 | 2024-04-03T22:21:40.714Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2539 | 2024-04-03T22:21:40.714Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 2540 | 2024-04-03T22:21:40.732Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2541 | 2024-04-03T22:21:40.732Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 2542 | 2024-04-03T22:21:40.750Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2543 | 2024-04-03T22:21:40.750Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 2544 | 2024-04-03T22:21:40.768Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2545 | 2024-04-03T22:21:40.768Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 2546 | 2024-04-03T22:21:40.786Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2547 | 2024-04-03T22:21:40.786Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 2548 | 2024-04-03T22:21:40.804Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2549 | 2024-04-03T22:21:40.804Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 2550 | 2024-04-03T22:21:40.821Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2551 | 2024-04-03T22:21:40.821Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 2552 | 2024-04-03T22:21:40.839Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2553 | 2024-04-03T22:21:40.839Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 2554 | 2024-04-03T22:21:40.856Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2555 | 2024-04-03T22:21:40.856Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2556 | 2024-04-03T22:21:40.856Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2557 | 2024-04-03T22:21:40.856Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2558 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2559 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2560 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2561 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2562 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2563 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2564 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2565 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2566 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2567 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2568 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2569 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2570 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2571 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2572 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 2573 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2574 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2575 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2576 | 2024-04-03T22:21:40.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3009
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2577 | 2024-04-03T22:21:40.946Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4", TimeNotSynchronized), ("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4", TimeNotSynchronized), ("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f", TimeNotSynchronized), ("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733", TimeNotSynchronized), ("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521", TimeNotSynchronized), ("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2", TimeNotSynchronized), ("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0", TimeNotSynchronized), ("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc", TimeNotSynchronized), ("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860", TimeNotSynchronized), ("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a", TimeNotSynchronized), ("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794", TimeNotSynchronized), ("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326", TimeNotSynchronized), ("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357", TimeNotSynchronized), ("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6", TimeNotSynchronized), ("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380", TimeNotSynchronized), ("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881", TimeNotSynchronized), ("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175", TimeNotSynchronized), ("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca", TimeNotSynchronized), ("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837
latency_us = 454044
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = 8a57c23f-2bed-43da-96dd-83f88b50ad93
response_code = 500
uri = /omicron-zones
|
| 2578 | 2024-04-03T22:21:40.947Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "8a57c23f-2bed-43da-96dd-83f88b50ad93", "content-length": "124", "date": "Wed, 03 Apr 2024 22:21:40 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "8a57c23f-2bed-43da-96dd-83f88b50ad93" }
file = sled-agent/src/rack_setup/service.rs:465
retry_after = 11.931497086s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2579 | 2024-04-03T22:21:52.911Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name("ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:427
sled_address = [fd00:1122:3344:101::1]:12345
|
| 2580 | 2024-04-03T22:21:52.912Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2581 | 2024-04-03T22:21:52.912Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }
|
| 2582 | 2024-04-03T22:21:52.931Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2583 | 2024-04-03T22:21:52.931Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }
|
| 2584 | 2024-04-03T22:21:52.947Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2585 | 2024-04-03T22:21:52.947Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }
|
| 2586 | 2024-04-03T22:21:52.966Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2587 | 2024-04-03T22:21:52.966Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2588 | 2024-04-03T22:21:52.984Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2589 | 2024-04-03T22:21:52.984Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2590 | 2024-04-03T22:21:53.004Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2591 | 2024-04-03T22:21:53.004Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2592 | 2024-04-03T22:21:53.024Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2593 | 2024-04-03T22:21:53.024Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2594 | 2024-04-03T22:21:53.042Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2595 | 2024-04-03T22:21:53.042Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }
|
| 2596 | 2024-04-03T22:21:53.060Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2597 | 2024-04-03T22:21:53.060Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2598 | 2024-04-03T22:21:53.080Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2599 | 2024-04-03T22:21:53.080Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }
|
| 2600 | 2024-04-03T22:21:53.098Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2601 | 2024-04-03T22:21:53.099Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }
|
| 2602 | 2024-04-03T22:21:53.117Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2603 | 2024-04-03T22:21:53.117Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }
|
| 2604 | 2024-04-03T22:21:53.134Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2605 | 2024-04-03T22:21:53.135Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }
|
| 2606 | 2024-04-03T22:21:53.153Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2607 | 2024-04-03T22:21:53.153Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }
|
| 2608 | 2024-04-03T22:21:53.171Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2609 | 2024-04-03T22:21:53.171Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }
|
| 2610 | 2024-04-03T22:21:53.190Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2611 | 2024-04-03T22:21:53.190Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }
|
| 2612 | 2024-04-03T22:21:53.208Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2613 | 2024-04-03T22:21:53.208Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }
|
| 2614 | 2024-04-03T22:21:53.227Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2615 | 2024-04-03T22:21:53.227Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }
|
| 2616 | 2024-04-03T22:21:53.244Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2617 | 2024-04-03T22:21:53.244Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }
|
| 2618 | 2024-04-03T22:21:53.263Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 2619 | 2024-04-03T22:21:53.263Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }
|
| 2620 | 2024-04-03T22:21:53.281Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2621 | 2024-04-03T22:21:53.281Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2622 | 2024-04-03T22:21:53.282Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2623 | 2024-04-03T22:21:53.282Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2624 | 2024-04-03T22:21:53.385Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:258
|
| 2625 | 2024-04-03T22:21:53.410Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:258
|
| 2626 | 2024-04-03T22:21:53.410Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2627 | 2024-04-03T22:21:53.410Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2628 | 2024-04-03T22:21:53.424Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:258
|
| 2629 | 2024-04-03T22:21:53.424Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2630 | 2024-04-03T22:21:53.437Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:258
|
| 2631 | 2024-04-03T22:21:53.437Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2632 | 2024-04-03T22:21:53.450Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:258
|
| 2633 | 2024-04-03T22:21:53.450Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2634 | 2024-04-03T22:21:53.463Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2635 | 2024-04-03T22:21:53.640Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2636 | 2024-04-03T22:21:53.640Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2637 | 2024-04-03T22:21:53.640Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2638 | 2024-04-03T22:21:53.640Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2639 | 2024-04-03T22:21:53.640Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2640 | 2024-04-03T22:21:53.640Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2641 | 2024-04-03T22:21:53.640Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2642 | 2024-04-03T22:21:53.640Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2643 | 2024-04-03T22:21:53.640Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
file = illumos-utils/src/zone.rs:315
|
| 2644 | 2024-04-03T22:21:53.645Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
file = illumos-utils/src/zone.rs:315
|
| 2645 | 2024-04-03T22:21:53.650Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
file = illumos-utils/src/zone.rs:315
|
| 2646 | 2024-04-03T22:21:53.655Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
file = illumos-utils/src/zone.rs:315
|
| 2647 | 2024-04-03T22:21:53.660Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
file = illumos-utils/src/zone.rs:315
|
| 2648 | 2024-04-03T22:21:53.665Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
file = illumos-utils/src/zone.rs:315
|
| 2649 | 2024-04-03T22:21:53.826Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
file = illumos-utils/src/zone.rs:354
|
| 2650 | 2024-04-03T22:21:53.831Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
file = illumos-utils/src/zone.rs:354
|
| 2651 | 2024-04-03T22:21:53.835Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
file = illumos-utils/src/zone.rs:354
|
| 2652 | 2024-04-03T22:21:53.840Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
file = illumos-utils/src/zone.rs:354
|
| 2653 | 2024-04-03T22:21:53.844Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
file = illumos-utils/src/zone.rs:354
|
| 2654 | 2024-04-03T22:21:53.849Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
file = illumos-utils/src/zone.rs:354
|
| 2655 | 2024-04-03T22:21:53.855Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
file = illumos-utils/src/zone.rs:315
|
| 2656 | 2024-04-03T22:21:53.860Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
file = illumos-utils/src/zone.rs:315
|
| 2657 | 2024-04-03T22:21:53.864Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
file = illumos-utils/src/zone.rs:315
|
| 2658 | 2024-04-03T22:21:53.869Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
file = illumos-utils/src/zone.rs:315
|
| 2659 | 2024-04-03T22:21:53.875Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
file = illumos-utils/src/zone.rs:315
|
| 2660 | 2024-04-03T22:21:53.880Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
file = illumos-utils/src/zone.rs:315
|
| 2661 | 2024-04-03T22:21:53.885Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
file = illumos-utils/src/zone.rs:315
|
| 2662 | 2024-04-03T22:21:53.891Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
file = illumos-utils/src/zone.rs:315
|
| 2663 | 2024-04-03T22:21:53.898Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
file = illumos-utils/src/zone.rs:354
|
| 2664 | 2024-04-03T22:21:53.904Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
file = illumos-utils/src/zone.rs:354
|
| 2665 | 2024-04-03T22:21:53.910Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
file = illumos-utils/src/zone.rs:354
|
| 2666 | 2024-04-03T22:21:53.915Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
file = illumos-utils/src/zone.rs:354
|
| 2667 | 2024-04-03T22:21:53.921Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
file = illumos-utils/src/zone.rs:354
|
| 2668 | 2024-04-03T22:21:53.926Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
file = illumos-utils/src/zone.rs:354
|
| 2669 | 2024-04-03T22:21:53.933Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
file = illumos-utils/src/zone.rs:354
|
| 2670 | 2024-04-03T22:21:53.938Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
file = illumos-utils/src/zone.rs:354
|
| 2671 | 2024-04-03T22:21:53.981Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:275
|
| 2672 | 2024-04-03T22:21:54.023Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:275
|
| 2673 | 2024-04-03T22:21:54.059Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:275
|
| 2674 | 2024-04-03T22:21:54.094Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:275
|
| 2675 | 2024-04-03T22:21:54.125Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:275
|
| 2676 | 2024-04-03T22:21:54.127Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:300
|
| 2677 | 2024-04-03T22:21:54.127Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:192
|
| 2678 | 2024-04-03T22:21:54.183Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte1", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 160, 169]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: "vopte1" } }
|
| 2679 | 2024-04-03T22:21:54.183Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1285
zone_type = external_dns
|
| 2680 | 2024-04-03T22:21:54.185Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:300
|
| 2681 | 2024-04-03T22:21:54.185Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:192
|
| 2682 | 2024-04-03T22:21:54.233Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte2", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 132, 230]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte2" } }
|
| 2683 | 2024-04-03T22:21:54.233Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1285
zone_type = nexus
|
| 2684 | 2024-04-03T22:21:54.234Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:300
|
| 2685 | 2024-04-03T22:21:54.234Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:192
|
| 2686 | 2024-04-03T22:21:54.287Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte3", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 153, 236]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte3" } }
|
| 2687 | 2024-04-03T22:21:54.287Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1285
zone_type = nexus
|
| 2688 | 2024-04-03T22:21:54.289Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:300
|
| 2689 | 2024-04-03T22:21:54.289Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:192
|
| 2690 | 2024-04-03T22:21:54.336Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte4", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 140, 131]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: "vopte4" } }
|
| 2691 | 2024-04-03T22:21:54.337Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1285
zone_type = external_dns
|
| 2692 | 2024-04-03T22:21:54.338Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:300
|
| 2693 | 2024-04-03T22:21:54.338Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:192
|
| 2694 | 2024-04-03T22:21:54.391Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte5", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 166, 2]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte5" } }
|
| 2695 | 2024-04-03T22:21:54.391Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1285
zone_type = nexus
|
| 2696 | 2024-04-03T22:21:54.392Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.1.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 2697 | 2024-04-03T22:21:54.392Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.1.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 2698 | 2024-04-03T22:21:54.392Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.1.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 2699 | 2024-04-03T22:21:54.392Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.1.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 2700 | 2024-04-03T22:21:54.393Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.1.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 2701 | 2024-04-03T22:21:54.393Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.1.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 2702 | 2024-04-03T22:21:54.394Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.1.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 2703 | 2024-04-03T22:21:54.394Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.1.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 2704 | 2024-04-03T22:21:54.394Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.1.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 2705 | 2024-04-03T22:21:54.397Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2706 | 2024-04-03T22:21:54.397Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.1.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 2707 | 2024-04-03T22:21:54.398Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.1.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 2708 | 2024-04-03T22:21:54.399Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2709 | 2024-04-03T22:21:54.399Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2710 | 2024-04-03T22:21:54.427Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.1.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 2711 | 2024-04-03T22:21:54.427Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.1.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 2712 | 2024-04-03T22:21:54.427Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.1.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 2713 | 2024-04-03T22:21:54.480Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2714 | 2024-04-03T22:21:54.480Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
file = illumos-utils/src/zone.rs:315
|
| 2715 | 2024-04-03T22:21:54.486Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.1.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 2716 | 2024-04-03T22:21:54.513Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
file = illumos-utils/src/zone.rs:315
|
| 2717 | 2024-04-03T22:21:54.518Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
file = illumos-utils/src/zone.rs:315
|
| 2718 | 2024-04-03T22:21:54.525Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2719 | 2024-04-03T22:21:54.525Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
file = illumos-utils/src/zone.rs:354
|
| 2720 | 2024-04-03T22:21:54.561Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
file = illumos-utils/src/zone.rs:315
|
| 2721 | 2024-04-03T22:21:54.567Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
file = illumos-utils/src/zone.rs:354
|
| 2722 | 2024-04-03T22:21:54.586Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
file = illumos-utils/src/zone.rs:354
|
| 2723 | 2024-04-03T22:21:54.602Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
file = illumos-utils/src/zone.rs:315
|
| 2724 | 2024-04-03T22:21:54.626Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
file = illumos-utils/src/zone.rs:354
|
| 2725 | 2024-04-03T22:21:54.649Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
file = illumos-utils/src/zone.rs:354
|
| 2726 | 2024-04-03T22:22:05.081Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService14"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::12"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_7d954909-2e42-44b1-96d9-f536366f6758/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::12"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="83782779-bae7-4273-babb-4a3a29feee57"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2727 | 2024-04-03T22:22:05.280Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService15"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::15"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_ae4e2e29-ff41-46eb-978b-335ffe5cd37e/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::15"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="1edb8c69-63bd-486a-bf78-4b0a86f1c0c8"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2728 | 2024-04-03T22:22:05.379Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService11"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::13"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_0c6b0894-9c77-422a-8363-2705812a672c/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::13"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="cc639a07-ec22-4794-9a8d-7ad4532ae956"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2729 | 2024-04-03T22:22:05.590Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService12"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::18"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_00b9d2d0-59a4-4682-aa18-072c6202c0c8/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::18"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="13556b43-b43d-4c18-aaf8-2e1f9e4ea50e"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2730 | 2024-04-03T22:22:05.591Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2731 | 2024-04-03T22:22:06.012Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2732 | 2024-04-03T22:22:06.226Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService16"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::17"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_494a3a1c-f913-4e4b-8eec-202ec6c5bfd4/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::17"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="1e06c0cc-c52a-47a7-a2a6-dea76d21bbc2"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2733 | 2024-04-03T22:22:06.227Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService17"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::19"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_70a16060-4587-476f-9772-16ea3a5a39aa/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::19"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="1701cfad-27e2-4647-b771-a24252ef32a8"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2734 | 2024-04-03T22:22:06.228Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService20"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::1a"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_761d3524-c333-4332-9563-7d1ceb36709e/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::1a"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="48c9c43d-4efc-4cb4-91dc-f77ace0f52b4"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2735 | 2024-04-03T22:22:06.229Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService22"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::16"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_4eb310e9-0d98-4a07-821c-54f35a32dc4a/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::16"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="db215025-90b0-40a8-8b8c-59d6b2779781"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2736 | 2024-04-03T22:22:06.229Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2737 | 2024-04-03T22:22:06.443Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2738 | 2024-04-03T22:22:06.451Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService13"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::11"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::11"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2739 | 2024-04-03T22:22:06.451Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2740 | 2024-04-03T22:22:06.458Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2741 | 2024-04-03T22:22:06.464Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2742 | 2024-04-03T22:22:06.470Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2743 | 2024-04-03T22:22:06.478Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService10"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::14"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_af9b6346-5d28-46df-baea-640dad9b76c3/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::14"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="94d7047c-7fc5-4784-97c6-af7b34682c45"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2744 | 2024-04-03T22:22:06.478Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2745 | 2024-04-03T22:22:06.484Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2746 | 2024-04-03T22:22:06.791Z | INFO | SledAgent (ServiceManager): Profile for oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService24"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::9"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte1"/>
<propval type="astring" name="gateway" value="172.30.1.1"/>
<propval type="astring" name="ip" value="172.30.1.6"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/external_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:101::9]:5353"/>
<propval type="astring" name="dns_address" value="172.30.1.6:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2747 | 2024-04-03T22:22:06.792Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2748 | 2024-04-03T22:22:08.554Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService28"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::a"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte2"/>
<propval type="astring" name="gateway" value="172.30.2.1"/>
<propval type="astring" name="ip" value="172.30.2.5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2749 | 2024-04-03T22:22:08.554Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService27"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::c"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte3"/>
<propval type="astring" name="gateway" value="172.30.2.1"/>
<propval type="astring" name="ip" value="172.30.2.7"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2750 | 2024-04-03T22:22:08.554Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2751 | 2024-04-03T22:22:08.559Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2752 | 2024-04-03T22:22:08.622Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 67.019838ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2753 | 2024-04-03T22:22:08.631Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 33.718615ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2754 | 2024-04-03T22:22:08.639Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 59.482152ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2755 | 2024-04-03T22:22:08.648Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 44.290517ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2756 | 2024-04-03T22:22:08.656Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 63.085948ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2757 | 2024-04-03T22:22:08.664Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 74.356222ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2758 | 2024-04-03T22:22:08.671Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 47.961483ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2759 | 2024-04-03T22:22:08.679Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 65.676896ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2760 | 2024-04-03T22:22:08.686Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 93.116466ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2761 | 2024-04-03T22:22:08.694Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 66.363192ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2762 | 2024-04-03T22:22:08.703Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 70.156815ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2763 | 2024-04-03T22:22:08.711Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 71.988097ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2764 | 2024-04-03T22:22:08.720Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 114.069506ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2765 | 2024-04-03T22:22:08.728Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 139.050083ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2766 | 2024-04-03T22:22:08.737Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 142.443111ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2767 | 2024-04-03T22:22:08.745Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 140.183759ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2768 | 2024-04-03T22:22:08.757Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 78.926929ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2769 | 2024-04-03T22:22:08.765Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 126.783248ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2770 | 2024-04-03T22:22:08.772Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 55.400782ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2771 | 2024-04-03T22:22:08.779Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 55.190854ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2772 | 2024-04-03T22:22:08.789Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 144.966992ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2773 | 2024-04-03T22:22:08.856Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 93.749633ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2774 | 2024-04-03T22:22:08.867Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 179.523061ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2775 | 2024-04-03T22:22:08.878Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 144.241357ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2776 | 2024-04-03T22:22:08.888Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 262.317266ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2777 | 2024-04-03T22:22:08.896Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 101.736708ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2778 | 2024-04-03T22:22:08.905Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 278.400889ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2779 | 2024-04-03T22:22:08.913Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 60.392927ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2780 | 2024-04-03T22:22:08.922Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 129.377531ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2781 | 2024-04-03T22:22:08.931Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 100.775581ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2782 | 2024-04-03T22:22:08.942Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 275.252927ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2783 | 2024-04-03T22:22:08.952Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 276.514148ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2784 | 2024-04-03T22:22:08.963Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 169.228588ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2785 | 2024-04-03T22:22:08.974Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 263.998356ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2786 | 2024-04-03T22:22:08.986Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 173.533646ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2787 | 2024-04-03T22:22:09.011Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 167.535529ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2788 | 2024-04-03T22:22:09.033Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 431.796374ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2789 | 2024-04-03T22:22:09.042Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 476.940745ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2790 | 2024-04-03T22:22:09.057Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 584.896294ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2791 | 2024-04-03T22:22:09.065Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 259.742812ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2792 | 2024-04-03T22:22:09.143Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 480.297905ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2793 | 2024-04-03T22:22:09.160Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 466.781678ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2794 | 2024-04-03T22:22:09.171Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 568.481061ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2795 | 2024-04-03T22:22:09.191Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 597.56072ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2796 | 2024-04-03T22:22:09.200Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 378.00546ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2797 | 2024-04-03T22:22:09.230Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 240.575769ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2798 | 2024-04-03T22:22:09.239Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 309.787268ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2799 | 2024-04-03T22:22:09.248Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 231.746844ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2800 | 2024-04-03T22:22:09.337Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 901.862405ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2801 | 2024-04-03T22:22:09.480Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 450.50676ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2802 | 2024-04-03T22:22:09.493Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 912.828054ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2803 | 2024-04-03T22:22:09.503Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 594.83205ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2804 | 2024-04-03T22:22:09.532Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 1.178214714s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2805 | 2024-04-03T22:22:09.563Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 829.622101ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2806 | 2024-04-03T22:22:09.590Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 617.977799ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2807 | 2024-04-03T22:22:09.636Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 1.099878491s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2808 | 2024-04-03T22:22:09.648Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 890.246779ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2809 | 2024-04-03T22:22:09.660Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 1.027693055s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2810 | 2024-04-03T22:22:09.753Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 1.166478344s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2811 | 2024-04-03T22:22:09.802Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 735.062997ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2812 | 2024-04-03T22:22:09.946Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 1.379355205s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2813 | 2024-04-03T22:22:10.110Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 778.990235ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2814 | 2024-04-03T22:22:10.221Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 1.488014333s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2815 | 2024-04-03T22:22:10.252Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 1.153305076s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2816 | 2024-04-03T22:22:10.404Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 1.461156628s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2817 | 2024-04-03T22:22:10.415Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 1.129867343s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2818 | 2024-04-03T22:22:10.475Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:48311
|
| 2819 | 2024-04-03T22:22:10.476Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 467
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:48311
req_id = c6b32bf3-f578-45af-a356-a1dea01ed563
response_code = 200
uri = /baseboard
|
| 2820 | 2024-04-03T22:22:10.546Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 1.180701454s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2821 | 2024-04-03T22:22:10.553Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 794.364362ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2822 | 2024-04-03T22:22:10.697Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 1.162384922s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2823 | 2024-04-03T22:22:10.720Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 1.262151682s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2824 | 2024-04-03T22:22:10.744Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 763.714891ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2825 | 2024-04-03T22:22:10.898Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 1.417790738s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2826 | 2024-04-03T22:22:10.929Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 1.127014951s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2827 | 2024-04-03T22:22:11.336Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 1.342575579s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2828 | 2024-04-03T22:22:11.356Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 1.174623439s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2829 | 2024-04-03T22:22:11.414Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 748.950899ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2830 | 2024-04-03T22:22:11.516Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 1.357564336s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2831 | 2024-04-03T22:22:11.552Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 1.065933184s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2832 | 2024-04-03T22:22:11.719Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 1.478262574s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2833 | 2024-04-03T22:22:11.736Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 1.011324745s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2834 | 2024-04-03T22:22:12.038Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 1.152698496s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2835 | 2024-04-03T22:22:12.254Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 844.844212ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2836 | 2024-04-03T22:22:12.607Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 1.181479916s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2837 | 2024-04-03T22:22:12.634Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 1.023583112s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2838 | 2024-04-03T22:22:12.663Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 1.444388044s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2839 | 2024-04-03T22:22:12.690Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 626.969075ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2840 | 2024-04-03T22:22:12.725Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 1.337931056s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2841 | 2024-04-03T22:22:12.746Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 785.532927ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2842 | 2024-04-03T22:22:12.763Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 783.175249ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2843 | 2024-04-03T22:22:12.782Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 970.11031ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2844 | 2024-04-03T22:22:12.895Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 979.193996ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2845 | 2024-04-03T22:22:13.115Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 1.065107415s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2846 | 2024-04-03T22:22:13.171Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 814.146316ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2847 | 2024-04-03T22:22:13.211Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 582.510155ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2848 | 2024-04-03T22:22:13.328Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 1.032576669s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2849 | 2024-04-03T22:22:13.544Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 536.556521ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2850 | 2024-04-03T22:22:13.559Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 1.448444697s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2851 | 2024-04-03T22:22:13.669Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 1.125765987s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2852 | 2024-04-03T22:22:13.762Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 730.575996ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2853 | 2024-04-03T22:22:13.799Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 983.812205ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2854 | 2024-04-03T22:22:13.808Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 1.003718389s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2855 | 2024-04-03T22:22:13.885Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 803.941518ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2856 | 2024-04-03T22:22:13.995Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 1.325659172s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2857 | 2024-04-03T22:22:14.072Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 636.079884ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2858 | 2024-04-03T22:22:14.089Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 1.10704557s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2859 | 2024-04-03T22:22:14.117Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 623.688644ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2860 | 2024-04-03T22:22:14.189Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 1.058809985s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2861 | 2024-04-03T22:22:14.371Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 968.084865ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2862 | 2024-04-03T22:22:14.503Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 597.037023ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2863 | 2024-04-03T22:22:14.698Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 1.075538072s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2864 | 2024-04-03T22:22:14.717Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 1.481859626s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2865 | 2024-04-03T22:22:14.750Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 1.49151222s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2866 | 2024-04-03T22:22:14.806Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 915.490179ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2867 | 2024-04-03T22:22:14.814Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 1.428371118s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2868 | 2024-04-03T22:22:14.822Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 728.482303ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2869 | 2024-04-03T22:22:15.017Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 1.136693736s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2870 | 2024-04-03T22:22:15.109Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 1.363924975s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2871 | 2024-04-03T22:22:15.205Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 1.443595416s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2872 | 2024-04-03T22:22:15.257Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 1.252623139s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2873 | 2024-04-03T22:22:15.330Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 809.562203ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2874 | 2024-04-03T22:22:15.349Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 1.471747772s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2875 | 2024-04-03T22:22:15.559Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 576.136579ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2876 | 2024-04-03T22:22:15.732Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 1.175522894s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2877 | 2024-04-03T22:22:15.783Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 1.285347397s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2878 | 2024-04-03T22:22:16.146Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 716.51786ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2879 | 2024-04-03T22:22:16.155Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 1.069887357s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2880 | 2024-04-03T22:22:16.164Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 1.482796802s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2881 | 2024-04-03T22:22:16.209Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 760.193919ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2882 | 2024-04-03T22:22:16.251Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 1.463107697s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2883 | 2024-04-03T22:22:16.259Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 627.304839ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2884 | 2024-04-03T22:22:16.482Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 1.485373841s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2885 | 2024-04-03T22:22:16.519Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 531.110508ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2886 | 2024-04-03T22:22:16.658Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 1.426339643s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2887 | 2024-04-03T22:22:16.830Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 1.197610587s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2888 | 2024-04-03T22:22:16.871Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 1.001843375s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2889 | 2024-04-03T22:22:16.894Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 785.27525ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2890 | 2024-04-03T22:22:16.917Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 1.36081872s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2891 | 2024-04-03T22:22:16.977Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 866.251029ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2892 | 2024-04-03T22:22:17.059Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 1.497207048s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2893 | 2024-04-03T22:22:17.077Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 1.451625902s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2894 | 2024-04-03T22:22:17.233Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 1.409008082s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2895 | 2024-04-03T22:22:17.657Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 1.458129601s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2896 | 2024-04-03T22:22:17.689Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 576.447187ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2897 | 2024-04-03T22:22:17.724Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 748.43462ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2898 | 2024-04-03T22:22:17.852Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 769.355436ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2899 | 2024-04-03T22:22:17.882Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 1.384799724s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2900 | 2024-04-03T22:22:17.976Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 1.095369492s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2901 | 2024-04-03T22:22:18.038Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 1.212992397s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2902 | 2024-04-03T22:22:18.093Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 1.086419569s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2903 | 2024-04-03T22:22:18.274Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 1.363878729s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2904 | 2024-04-03T22:22:18.285Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 1.104490993s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2905 | 2024-04-03T22:22:18.484Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 1.096879608s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2906 | 2024-04-03T22:22:18.539Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 568.056615ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2907 | 2024-04-03T22:22:18.565Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 1.245773803s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2908 | 2024-04-03T22:22:18.632Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 1.347432985s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2909 | 2024-04-03T22:22:18.651Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 1.009379826s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2910 | 2024-04-03T22:22:18.860Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService18"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::f"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::f"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2911 | 2024-04-03T22:22:18.861Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2912 | 2024-04-03T22:22:18.868Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService19"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::10"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::10"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2913 | 2024-04-03T22:22:18.868Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2914 | 2024-04-03T22:22:18.933Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357") failed: Property not found. retry in 34.615635ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2915 | 2024-04-03T22:22:18.981Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357") failed: Property not found. retry in 63.190641ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2916 | 2024-04-03T22:22:19.014Z | INFO | SledAgent (ServiceManager): Profile for oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService25"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::8"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte4"/>
<propval type="astring" name="gateway" value="172.30.1.1"/>
<propval type="astring" name="ip" value="172.30.1.5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/external_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:101::8]:5353"/>
<propval type="astring" name="dns_address" value="172.30.1.5:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2917 | 2024-04-03T22:22:19.014Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2918 | 2024-04-03T22:22:19.061Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357") failed: Property not found. retry in 202.79155ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2919 | 2024-04-03T22:22:19.092Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 1.18256325s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2920 | 2024-04-03T22:22:19.121Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 842.007039ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2921 | 2024-04-03T22:22:19.132Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 861.472176ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2922 | 2024-04-03T22:22:19.157Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f") failed: Property not found. retry in 32.694258ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2923 | 2024-04-03T22:22:19.167Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733") failed: Property not found. retry in 74.53486ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2924 | 2024-04-03T22:22:19.189Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 1.303859122s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2925 | 2024-04-03T22:22:19.198Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f") failed: Property not found. retry in 101.833236ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2926 | 2024-04-03T22:22:19.252Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733") failed: Property not found. retry in 89.23912ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2927 | 2024-04-03T22:22:19.262Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 714.069751ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2928 | 2024-04-03T22:22:19.270Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6") failed: Property not found. retry in 49.635704ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2929 | 2024-04-03T22:22:19.278Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 1.075150276s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2930 | 2024-04-03T22:22:19.285Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357") failed: Property not found. retry in 534.179613ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2931 | 2024-04-03T22:22:19.308Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f") failed: Property not found. retry in 186.270268ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2932 | 2024-04-03T22:22:19.328Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6") failed: Property not found. retry in 55.395625ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2933 | 2024-04-03T22:22:19.350Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733") failed: Property not found. retry in 249.387781ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2934 | 2024-04-03T22:22:19.392Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6") failed: Property not found. retry in 216.160392ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2935 | 2024-04-03T22:22:19.401Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 1.018969291s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2936 | 2024-04-03T22:22:19.498Z | INFO | SledAgent (ServiceManager): Profile for oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService23"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::d"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/oximeter">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="id" value="2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"/>
<propval type="astring" name="address" value="[fd00:1122:3344:101::d]:12223"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2937 | 2024-04-03T22:22:19.507Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f") failed: Property not found. retry in 499.493639ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2938 | 2024-04-03T22:22:19.507Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2939 | 2024-04-03T22:22:19.594Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 1.209054429s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2940 | 2024-04-03T22:22:19.624Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733") failed: Property not found. retry in 542.151153ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2941 | 2024-04-03T22:22:19.655Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6") failed: Property not found. retry in 518.241167ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2942 | 2024-04-03T22:22:19.679Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 965.877623ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2943 | 2024-04-03T22:22:19.691Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 1.045738388s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2944 | 2024-04-03T22:22:19.827Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 987.330072ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2945 | 2024-04-03T22:22:19.841Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357") failed: Property not found. retry in 1.053667243s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2946 | 2024-04-03T22:22:19.853Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4") failed: Property not found. retry in 64.809713ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2947 | 2024-04-03T22:22:19.899Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService26"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::b"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte5"/>
<propval type="astring" name="gateway" value="172.30.2.1"/>
<propval type="astring" name="ip" value="172.30.2.6"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2948 | 2024-04-03T22:22:19.899Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2949 | 2024-04-03T22:22:19.927Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4") failed: Property not found. retry in 73.94577ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2950 | 2024-04-03T22:22:19.976Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 725.699185ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2951 | 2024-04-03T22:22:19.986Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 1.072313842s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2952 | 2024-04-03T22:22:19.995Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 1.043573208s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2953 | 2024-04-03T22:22:20.003Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 522.811046ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2954 | 2024-04-03T22:22:20.010Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4") failed: Property not found. retry in 212.831512ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2955 | 2024-04-03T22:22:20.019Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f") failed: Property not found. retry in 447.394475ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2956 | 2024-04-03T22:22:20.080Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0") failed: Property not found. retry in 60.410887ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2957 | 2024-04-03T22:22:20.150Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0") failed: Property not found. retry in 118.600264ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2958 | 2024-04-03T22:22:20.176Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733") failed: Property not found. retry in 1.131534091s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2959 | 2024-04-03T22:22:20.186Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6") failed: Property not found. retry in 1.075565782s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2960 | 2024-04-03T22:22:20.232Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4") failed: Property not found. retry in 520.592428ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2961 | 2024-04-03T22:22:20.281Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0") failed: Property not found. retry in 213.096436ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2962 | 2024-04-03T22:22:20.290Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 1.059274268s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2963 | 2024-04-03T22:22:20.367Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 1.334181933s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2964 | 2024-04-03T22:22:20.433Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 818.330595ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2965 | 2024-04-03T22:22:20.477Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f") failed: Property not found. retry in 1.093729783s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2966 | 2024-04-03T22:22:20.503Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 623.861764ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2967 | 2024-04-03T22:22:20.512Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0") failed: Property not found. retry in 264.136861ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2968 | 2024-04-03T22:22:20.536Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 1.167685319s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2969 | 2024-04-03T22:22:20.657Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 547.304158ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2970 | 2024-04-03T22:22:20.713Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 871.34397ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2971 | 2024-04-03T22:22:20.752Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 547.718347ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2972 | 2024-04-03T22:22:20.769Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4") failed: Property not found. retry in 1.17870537s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2973 | 2024-04-03T22:22:20.790Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0") failed: Property not found. retry in 798.181965ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2974 | 2024-04-03T22:22:20.817Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 530.879076ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2975 | 2024-04-03T22:22:20.831Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 1.273164552s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2976 | 2024-04-03T22:22:20.910Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357") failed: Property not found. retry in 842.33271ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2977 | 2024-04-03T22:22:21.051Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 709.793419ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2978 | 2024-04-03T22:22:21.073Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 795.093517ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2979 | 2024-04-03T22:22:21.147Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 1.471139367s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 2980 | 2024-04-03T22:22:21.224Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 804.752043ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2981 | 2024-04-03T22:22:21.264Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 1.268091805s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 2982 | 2024-04-03T22:22:21.276Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6") failed: Property not found. retry in 1.026726057s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 2983 | 2024-04-03T22:22:21.314Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 1.269947126s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 2984 | 2024-04-03T22:22:21.326Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733") failed: Property not found. retry in 1.473357959s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 2985 | 2024-04-03T22:22:21.361Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 818.17529ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 2986 | 2024-04-03T22:22:21.375Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 587.766459ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2987 | 2024-04-03T22:22:21.584Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f") failed: Property not found. retry in 712.285567ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 2988 | 2024-04-03T22:22:21.598Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 1.020436222s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 2989 | 2024-04-03T22:22:21.610Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0") failed: Property not found. retry in 563.123992ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 2990 | 2024-04-03T22:22:21.715Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 1.131699464s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 2991 | 2024-04-03T22:22:21.728Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 1.159982376s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 2992 | 2024-04-03T22:22:21.766Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357") failed: Property not found. retry in 1.242901518s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 2993 | 2024-04-03T22:22:21.778Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 1.470454193s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 2994 | 2024-04-03T22:22:21.882Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 1.490012847s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 2995 | 2024-04-03T22:22:21.963Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4") failed: Property not found. retry in 934.86602ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 2996 | 2024-04-03T22:22:21.977Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 885.58961ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 2997 | 2024-04-03T22:22:22.042Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 1.466190271s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 2998 | 2024-04-03T22:22:22.119Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 548.255449ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 2999 | 2024-04-03T22:22:22.188Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0") failed: Property not found. retry in 839.186122ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 3000 | 2024-04-03T22:22:22.201Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 883.43612ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 3001 | 2024-04-03T22:22:22.312Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f") failed: Property not found. retry in 1.269508286s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 3002 | 2024-04-03T22:22:22.326Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6") failed: Property not found. retry in 606.326134ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 3003 | 2024-04-03T22:22:22.546Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 1.317768267s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 3004 | 2024-04-03T22:22:22.599Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 1.079978432s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 3005 | 2024-04-03T22:22:22.633Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 708.443194ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 3006 | 2024-04-03T22:22:22.645Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 1.172728748s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 3007 | 2024-04-03T22:22:22.680Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 1.043871441s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 3008 | 2024-04-03T22:22:22.820Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733") failed: Property not found. retry in 633.241515ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 3009 | 2024-04-03T22:22:22.864Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 1.491046399s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 3010 | 2024-04-03T22:22:22.880Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 1.11798s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 3011 | 2024-04-03T22:22:22.904Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 875.366712ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 3012 | 2024-04-03T22:22:22.916Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4") failed: Property not found. retry in 504.689551ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 3013 | 2024-04-03T22:22:22.948Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6") failed: Property not found. retry in 1.116226694s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 3014 | 2024-04-03T22:22:23.022Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357") failed: Property not found. retry in 757.619445ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 3015 | 2024-04-03T22:22:23.043Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0") failed: Property not found. retry in 965.703097ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 3016 | 2024-04-03T22:22:23.101Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 1.411247556s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 3017 | 2024-04-03T22:22:23.263Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 877.454993ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 3018 | 2024-04-03T22:22:23.358Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 979.025993ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 3019 | 2024-04-03T22:22:23.388Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 896.539167ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 3020 | 2024-04-03T22:22:23.435Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4") failed: Property not found. retry in 1.243601129s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 3021 | 2024-04-03T22:22:23.451Z | INFO | SledAgent (ServiceManager): Profile for oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService21"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::e"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/clickhouse">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::e"/>
<propval type="astring" name="listen_port" value="8123"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 3022 | 2024-04-03T22:22:23.469Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733") failed: Property not found. retry in 1.178567315s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 3023 | 2024-04-03T22:22:23.469Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3024 | 2024-04-03T22:22:23.523Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 869.495205ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 3025 | 2024-04-03T22:22:23.596Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f") failed: Property not found. retry in 681.314701ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 3026 | 2024-04-03T22:22:23.695Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 845.392862ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 3027 | 2024-04-03T22:22:23.739Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 1.46242182s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 3028 | 2024-04-03T22:22:23.776Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 32.229291ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3029 | 2024-04-03T22:22:23.792Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 739.442569ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 3030 | 2024-04-03T22:22:23.802Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357") failed: Property not found. retry in 1.327773812s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 3031 | 2024-04-03T22:22:23.821Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 74.881629ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3032 | 2024-04-03T22:22:23.835Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 1.363186644s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 3033 | 2024-04-03T22:22:23.878Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 1.321754467s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 3034 | 2024-04-03T22:22:23.909Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 217.229975ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3035 | 2024-04-03T22:22:24.018Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 1.429455835s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 3036 | 2024-04-03T22:22:24.037Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0") failed: Property not found. retry in 760.482489ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 3037 | 2024-04-03T22:22:24.085Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6") failed: Property not found. retry in 1.421151714s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 3038 | 2024-04-03T22:22:24.148Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 551.976074ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3039 | 2024-04-03T22:22:24.168Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 1.057408296s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 3040 | 2024-04-03T22:22:24.290Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f") failed: Property not found. retry in 636.45835ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 3041 | 2024-04-03T22:22:24.302Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 1.270480978s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 3042 | 2024-04-03T22:22:24.349Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 1.091517551s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 3043 | 2024-04-03T22:22:24.369Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 1.229169142s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 3044 | 2024-04-03T22:22:24.406Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 848.524835ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 3045 | 2024-04-03T22:22:24.526Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 1.019355335s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 3046 | 2024-04-03T22:22:24.544Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 1.442861107s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 3047 | 2024-04-03T22:22:24.557Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 1.414993551s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 3048 | 2024-04-03T22:22:24.658Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733") failed: Property not found. retry in 1.366454316s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 3049 | 2024-04-03T22:22:24.690Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4") failed: Property not found. retry in 570.558355ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 3050 | 2024-04-03T22:22:24.709Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 1.182628804s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3051 | 2024-04-03T22:22:24.810Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0") failed: Property not found. retry in 1.423371882s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 3052 | 2024-04-03T22:22:24.939Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f") failed: Property not found. retry in 618.561986ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 3053 | 2024-04-03T22:22:25.144Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357") failed: Property not found. retry in 1.172661965s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 3054 | 2024-04-03T22:22:25.214Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 718.170775ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 3055 | 2024-04-03T22:22:25.229Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 1.299552209s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 3056 | 2024-04-03T22:22:25.243Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 526.74354ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 3057 | 2024-04-03T22:22:25.256Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 933.248845ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 3058 | 2024-04-03T22:22:25.270Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 1.45914288s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 3059 | 2024-04-03T22:22:25.282Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4") failed: Property not found. retry in 1.454501628s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 3060 | 2024-04-03T22:22:25.460Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 1.154092939s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 3061 | 2024-04-03T22:22:25.478Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 1.112627718s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 3062 | 2024-04-03T22:22:25.524Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6") failed: Property not found. retry in 794.05123ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 3063 | 2024-04-03T22:22:25.559Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 864.624497ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 3064 | 2024-04-03T22:22:25.573Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f") failed: Property not found. retry in 1.471895329s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 3065 | 2024-04-03T22:22:25.585Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 1.154143983s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 3066 | 2024-04-03T22:22:25.610Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 789.485556ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 3067 | 2024-04-03T22:22:25.785Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 1.487213315s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 3068 | 2024-04-03T22:22:25.907Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 774.206631ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3069 | 2024-04-03T22:22:25.946Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 1.363233822s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 3070 | 2024-04-03T22:22:25.989Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 660.704389ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 3071 | 2024-04-03T22:22:26.002Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 1.346755255s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 3072 | 2024-04-03T22:22:26.038Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733") failed: Property not found. retry in 813.803974ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 3073 | 2024-04-03T22:22:26.205Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 826.625642ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 3074 | 2024-04-03T22:22:26.244Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0") failed: Property not found. retry in 549.833924ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 3075 | 2024-04-03T22:22:26.330Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357") failed: Property not found. retry in 827.706667ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 3076 | 2024-04-03T22:22:26.342Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6") failed: Property not found. retry in 549.659976ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 3077 | 2024-04-03T22:22:26.415Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 633.085981ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 3078 | 2024-04-03T22:22:26.439Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 1.298254285s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 3079 | 2024-04-03T22:22:26.544Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 1.362847819s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 3080 | 2024-04-03T22:22:26.603Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 1.189733636s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 3081 | 2024-04-03T22:22:26.627Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 906.571871ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 3082 | 2024-04-03T22:22:26.667Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 998.129817ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 3083 | 2024-04-03T22:22:26.698Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 1.359846792s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3084 | 2024-04-03T22:22:26.743Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326") failed: Property not found. retry in 1.199671715s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 3085 | 2024-04-03T22:22:26.758Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 539.271784ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 3086 | 2024-04-03T22:22:26.775Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4") failed: Property not found. retry in 766.103883ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 3087 | 2024-04-03T22:22:26.809Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0") failed: Property not found. retry in 681.187271ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 3088 | 2024-04-03T22:22:26.864Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733") failed: Property not found. retry in 548.432532ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 3089 | 2024-04-03T22:22:26.905Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6") failed: Property not found. retry in 1.19649774s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 3090 | 2024-04-03T22:22:27.045Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 615.389191ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 3091 | 2024-04-03T22:22:27.060Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f") failed: Property not found. retry in 858.954531ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 3092 | 2024-04-03T22:22:27.069Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794") failed: Property not found. retry in 872.467776ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 3093 | 2024-04-03T22:22:27.174Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357") failed: Property not found. retry in 908.794147ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 3094 | 2024-04-03T22:22:27.284Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 614.550073ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 3095 | 2024-04-03T22:22:27.311Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 635.949337ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 3096 | 2024-04-03T22:22:27.321Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a") failed: Property not found. retry in 815.637292ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 3097 | 2024-04-03T22:22:27.361Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc") failed: Property not found. retry in 949.81425ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 3098 | 2024-04-03T22:22:27.424Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733") failed: Property not found. retry in 812.682393ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 3099 | 2024-04-03T22:22:27.502Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0") failed: Property not found. retry in 507.846853ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 3100 | 2024-04-03T22:22:27.545Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175") failed: Property not found. retry in 864.018456ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 3101 | 2024-04-03T22:22:27.560Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4") failed: Property not found. retry in 1.40157717s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 3102 | 2024-04-03T22:22:27.675Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881") failed: Property not found. retry in 526.012031ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 3103 | 2024-04-03T22:22:27.687Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8") failed: Property not found. retry in 1.312959727s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 3104 | 2024-04-03T22:22:27.750Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860") failed: Property not found. retry in 1.399239159s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 3105 | 2024-04-03T22:22:27.813Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2") failed: Property not found. retry in 985.449898ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 3106 | 2024-04-03T22:22:27.919Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca") failed: Property not found. retry in 1.162750581s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 3107 | 2024-04-03T22:22:27.937Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4") failed: Property not found. retry in 781.910839ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 3108 | 2024-04-03T22:22:27.952Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f") failed: Property not found. retry in 1.187165712s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 3109 | 2024-04-03T22:22:28.019Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521") failed: Property not found. retry in 1.058381215s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 3110 | 2024-04-03T22:22:28.036Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0") failed: Property not found. retry in 1.184002006s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 3111 | 2024-04-03T22:22:28.076Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 1.218400801s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3112 | 2024-04-03T22:22:28.103Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357") failed: Property not found. retry in 1.129014505s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 3113 | 2024-04-03T22:22:28.131Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6") failed: Property not found. retry in 713.195568ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 3114 | 2024-04-03T22:22:28.262Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733") failed: Property not found. retry in 633.773408ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 3115 | 2024-04-03T22:22:28.929Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733") failed: Property not found. retry in 911.765234ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 3116 | 2024-04-03T22:22:29.487Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 1.284307797s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3117 | 2024-04-03T22:22:30.790Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 1.191186377s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3118 | 2024-04-03T22:22:31.994Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 504.354272ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3119 | 2024-04-03T22:22:32.509Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 1.275784762s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3120 | 2024-04-03T22:22:33.796Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 808.3738ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3121 | 2024-04-03T22:22:34.614Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 1.387253362s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3122 | 2024-04-03T22:22:36.014Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 563.630345ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3123 | 2024-04-03T22:22:36.588Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 936.144664ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3124 | 2024-04-03T22:22:37.535Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 633.30536ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3125 | 2024-04-03T22:22:38.180Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 936.362498ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3126 | 2024-04-03T22:22:39.127Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 590.866697ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3127 | 2024-04-03T22:22:39.728Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 519.144886ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3128 | 2024-04-03T22:22:40.258Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 686.131689ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3129 | 2024-04-03T22:22:40.508Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:33282
|
| 3130 | 2024-04-03T22:22:40.508Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 85
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:33282
req_id = 0ca62d66-c6d3-4331-bc00-43b0aabd5485
response_code = 200
uri = /baseboard
|
| 3131 | 2024-04-03T22:22:40.955Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 796.080924ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3132 | 2024-04-03T22:22:41.762Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 625.808231ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3133 | 2024-04-03T22:22:42.397Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380") failed: Property not found. retry in 979.665547ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3134 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8
|
| 3135 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175
|
| 3136 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc
|
| 3137 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2
|
| 3138 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794
|
| 3139 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4
|
| 3140 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326
|
| 3141 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6
|
| 3142 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357
|
| 3143 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0
|
| 3144 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860
|
| 3145 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a
|
| 3146 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521
|
| 3147 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f
|
| 3148 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733
|
| 3149 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881
|
| 3150 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380
|
| 3151 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca
|
| 3152 | 2024-04-03T22:22:43.407Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3005
zone = oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4
|
| 3153 | 2024-04-03T22:22:43.409Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 3154 | 2024-04-03T22:22:43.410Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 3155 | 2024-04-03T22:22:43.414Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 50456987
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:53174
req_id = c1d5fa6b-6750-4827-b21e-21e70ece4669
response_code = 204
uri = /omicron-zones
|
| 3156 | 2024-04-03T22:22:43.438Z | INFO | SledAgent (RSS): Finished setting up services
file = sled-agent/src/rack_setup/service.rs:1230
|
| 3157 | 2024-04-03T22:22:43.438Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.rss-plan-completed.marker.tmp
file = common/src/ledger.rs:199
|
| 3158 | 2024-04-03T22:22:43.438Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.rss-plan-completed.marker.tmp
file = common/src/ledger.rs:199
|
| 3159 | 2024-04-03T22:22:43.442Z | INFO | SledAgent (RSS): Handing off control to Nexus
file = sled-agent/src/rack_setup/service.rs:677
|
| 3160 | 2024-04-03T22:22:43.444Z | INFO | SledAgent (RSS): Nexus address: [fd00:1122:3344:101::b]:12221
file = sled-agent/src/rack_setup/service.rs:685
|
| 3161 | 2024-04-03T22:22:43.484Z | INFO | SledAgent (RSS): rack_network_config: RackNetworkConfigV1 {
bfd: [],
bgp: [],
infra_ip_first: 10.151.1.174,
infra_ip_last: 10.151.1.174,
ports: [
PortConfigV1 {
addresses: [
V4(
Ipv4Network {
addr: 10.151.1.174,
prefix: 24,
},
),
],
autoneg: false,
bgp_peers: [],
port: "qsfp0",
routes: [
RouteConfig {
destination: V4(
Ipv4Network {
addr: 0.0.0.0,
prefix: 0,
},
),
nexthop: 10.151.1.100,
},
],
switch: Switch0,
uplink_port_fec: None,
uplink_port_speed: Speed40G,
},
],
rack_subnet: Ipv6Network {
addr: fd00:1122:3344:100::,
prefix: 56,
},
}
file = sled-agent/src/rack_setup/service.rs:816
|
| 3162 | 2024-04-03T22:22:43.637Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "772ddc8e-1c76-4d05-8e34-69d50676452f", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:43 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "772ddc8e-1c76-4d05-8e34-69d50676452f" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3163 | 2024-04-03T22:22:43.752Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "655bf0bb-3e6b-4cc5-b118-eea4cb4e352c", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:43 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "655bf0bb-3e6b-4cc5-b118-eea4cb4e352c" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3164 | 2024-04-03T22:22:43.973Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e995cf1c-0c09-4b9a-87ef-76189881d115", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:43 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e995cf1c-0c09-4b9a-87ef-76189881d115" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3165 | 2024-04-03T22:22:44.206Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "37f5ec01-1a58-44ad-b26b-25317124110f", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:43 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "37f5ec01-1a58-44ad-b26b-25317124110f" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3166 | 2024-04-03T22:22:44.516Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "c5594691-b1df-4145-ba29-f9cd3274c2b7", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:43 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "c5594691-b1df-4145-ba29-f9cd3274c2b7" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3167 | 2024-04-03T22:22:44.881Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6704d157-b564-4fa1-a59c-d865328a6f6b", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:44 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6704d157-b564-4fa1-a59c-d865328a6f6b" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3168 | 2024-04-03T22:22:45.261Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b0652bc7-ea93-45c7-bedc-665c3c5994d3", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:45 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b0652bc7-ea93-45c7-bedc-665c3c5994d3" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3169 | 2024-04-03T22:22:45.624Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e45e7e61-fae7-4c18-9899-37c4807ff2d1", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:44 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e45e7e61-fae7-4c18-9899-37c4807ff2d1" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3170 | 2024-04-03T22:22:45.959Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "ee3f7043-6c0e-4d8f-829c-77117ff2645c", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:45 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "ee3f7043-6c0e-4d8f-829c-77117ff2645c" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3171 | 2024-04-03T22:22:46.334Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "327a1541-ec25-42da-b567-dea630f66f42", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:46 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "327a1541-ec25-42da-b567-dea630f66f42" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3172 | 2024-04-03T22:22:47.097Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "afbfb227-9790-411f-969e-16944633296d", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:47 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "afbfb227-9790-411f-969e-16944633296d" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3173 | 2024-04-03T22:22:47.850Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "8ecac43d-0ef3-4121-a0c2-9a64a3130561", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:47 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "8ecac43d-0ef3-4121-a0c2-9a64a3130561" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3174 | 2024-04-03T22:22:48.347Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d99777c1-145c-4457-839e-53e222bfbff9", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:47 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d99777c1-145c-4457-839e-53e222bfbff9" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3175 | 2024-04-03T22:22:49.371Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "fd7f60e8-c972-477d-ad24-d967f3f28795", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:49 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "fd7f60e8-c972-477d-ad24-d967f3f28795" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3176 | 2024-04-03T22:22:50.352Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f2e159bb-77da-4cfe-84da-5034c657fe4a", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:49 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f2e159bb-77da-4cfe-84da-5034c657fe4a" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3177 | 2024-04-03T22:22:51.915Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "04ecaa8a-edc0-4e8f-85ed-e7a1a5b64006", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:51 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "04ecaa8a-edc0-4e8f-85ed-e7a1a5b64006" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3178 | 2024-04-03T22:22:53.589Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "73a3e20f-4843-4e34-9426-589424f20988", "content-length": "124", "date": "Wed, 03 Apr 2024 22:22:53 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "73a3e20f-4843-4e34-9426-589424f20988" }
file = sled-agent/src/rack_setup/service.rs:873
|
| 3179 | 2024-04-03T22:22:55.348Z | INFO | SledAgent (NexusNotifierTask): Successfully put SledAgentInfo to nexus
file = sled-agent/src/nexus.rs:592
new_generation = 1
old_generation = 1
sled_id = 1ee9e643-97de-413d-8b33-615f41c711a3
|
| 3180 | 2024-04-03T22:22:55.678Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:44378
|
| 3181 | 2024-04-03T22:22:55.680Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 1577
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:44378
req_id = f8258a8b-839f-4cba-a230-d97f0036ade7
response_code = 200
uri = /network-bootstore-config
|
| 3182 | 2024-04-03T22:22:56.436Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:44437
|
| 3183 | 2024-04-03T22:22:56.439Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name("external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name("external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name("nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name("nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name("nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 3184 | 2024-04-03T22:22:56.439Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 3185 | 2024-04-03T22:22:56.439Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 3186 | 2024-04-03T22:22:56.439Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 3187 | 2024-04-03T22:22:56.439Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 3188 | 2024-04-03T22:22:56.439Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 3189 | 2024-04-03T22:22:56.439Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 3190 | 2024-04-03T22:22:56.439Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 2194
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:44437
req_id = 81e0df6a-544a-499e-97c0-8f9fe4628a38
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 3191 | 2024-04-03T22:22:56.440Z | INFO | SledAgent (RSS): Handoff to Nexus is complete
file = sled-agent/src/rack_setup/service.rs:883
|
| 3192 | 2024-04-03T22:22:56.736Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:47220
|
| 3193 | 2024-04-03T22:22:56.738Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 3194 | 2024-04-03T22:22:56.825Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 88206
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:47220
req_id = b8f4a2bb-f69e-4012-a06c-7e2dfd97c06d
response_code = 200
uri = /inventory
|
| 3195 | 2024-04-03T22:22:56.826Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 3196 | 2024-04-03T22:22:56.826Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 3197 | 2024-04-03T22:22:56.827Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 3198 | 2024-04-03T22:22:56.828Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 2551
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:47220
req_id = eb993dc0-7939-4502-aa1e-daad260b2d65
response_code = 200
uri = /omicron-zones
|
| 3199 | 2024-04-03T22:23:10.546Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:33310
|
| 3200 | 2024-04-03T22:23:10.547Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 67
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:33310
req_id = 7b85a6e1-9064-4b19-abdf-3ce70f8a18cd
response_code = 200
uri = /baseboard
|
| 3201 | 2024-04-03T22:23:10.689Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:59506
|
| 3202 | 2024-04-03T22:23:10.689Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:56622
|
| 3203 | 2024-04-03T22:23:10.782Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 92448
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:59506
req_id = 99600225-e8e1-438b-b650-d080b95eb983
response_code = 204
uri = /switch-ports
|
| 3204 | 2024-04-03T22:23:10.785Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 436
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:59506
req_id = 86bf67b4-fde7-4318-a224-e5f4ee0902c9
response_code = 200
uri = /network-bootstore-config
|
| 3205 | 2024-04-03T22:23:10.808Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 2, current_generation: 1
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-buskin-i86pc
|
| 3206 | 2024-04-03T22:23:10.808Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-buskin-i86pc
|
| 3207 | 2024-04-03T22:23:10.810Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-buskin-i86pc
|
| 3208 | 2024-04-03T22:23:10.813Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 2
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-buskin-i86pc
|
| 3209 | 2024-04-03T22:23:10.813Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 6732
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:59506
req_id = 712aa250-4d81-4c33-82dc-0965d3c68df4
response_code = 204
uri = /network-bootstore-config
|
| 3210 | 2024-04-03T22:23:10.836Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 146478
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:56622
req_id = ffb5b67a-bf9b-48d8-9ba5-2d0f84aa1faf
response_code = 204
uri = /switch-ports
|
| 3211 | 2024-04-03T22:23:10.836Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 113
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:56622
req_id = f036b031-aaac-419e-8ec8-913770800b46
response_code = 200
uri = /network-bootstore-config
|
| 3212 | 2024-04-03T22:23:11.058Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:35225
|
| 3213 | 2024-04-03T22:23:11.103Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 44801
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:35225
req_id = 423b2eac-2ea7-4907-995f-893ae8eb2385
response_code = 204
uri = /switch-ports
|
| 3214 | 2024-04-03T22:23:11.103Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 89
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:35225
req_id = e0b5d09d-7df2-430c-ac59-204255e1673c
response_code = 200
uri = /network-bootstore-config
|
| 3215 | 2024-04-03T22:23:23.381Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::d]:57652
|
| 3216 | 2024-04-03T22:23:23.388Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 6458
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:57652
req_id = 472c3a5d-1ee1-4bf6-b7cd-bbe18be8370f
response_code = 200
uri = /metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3
|
| 3217 | 2024-04-03T22:23:40.580Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:61366
|
| 3218 | 2024-04-03T22:23:40.580Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 87
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:61366
req_id = ee505a09-1212-4595-97fc-1136fc40e529
response_code = 200
uri = /baseboard
|
| 3219 | 2024-04-03T22:23:40.866Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:54306
|
| 3220 | 2024-04-03T22:23:40.916Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 49976
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:54306
req_id = f24da286-98da-4e67-9cf7-0ae9f51ee4c4
response_code = 204
uri = /switch-ports
|
| 3221 | 2024-04-03T22:23:40.916Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 98
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:54306
req_id = ff2cb584-6c6b-4349-8dd9-afe4ae001461
response_code = 200
uri = /network-bootstore-config
|
| 3222 | 2024-04-03T22:23:40.929Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:47499
|
| 3223 | 2024-04-03T22:23:40.973Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 43548
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:47499
req_id = 5ee641e9-913c-4c88-9e58-57f125803919
response_code = 204
uri = /switch-ports
|
| 3224 | 2024-04-03T22:23:40.973Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 95
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:47499
req_id = 4fc8ba47-d17d-45f5-8485-b9feb9875ac6
response_code = 200
uri = /network-bootstore-config
|
| 3225 | 2024-04-03T22:23:41.414Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:63974
|
| 3226 | 2024-04-03T22:23:41.458Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 44119
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:63974
req_id = 683b9725-339f-475a-babf-13c49d1d1ec4
response_code = 204
uri = /switch-ports
|
| 3227 | 2024-04-03T22:23:41.459Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 691
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:63974
req_id = bb4c3d4d-826c-47f7-a491-ee20c9882d4e
response_code = 200
uri = /network-bootstore-config
|
| 3228 | 2024-04-03T22:23:53.383Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 2419
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:57652
req_id = 6105753b-cc41-4599-8566-c5c5d9969176
response_code = 200
uri = /metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3
|
| 3229 | 2024-04-03T22:24:10.616Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:59483
|
| 3230 | 2024-04-03T22:24:10.616Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 75
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:59483
req_id = cde693bc-76e8-4bc6-8e1b-78e1ab66a386
response_code = 200
uri = /baseboard
|
| 3231 | 2024-04-03T22:24:10.676Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:37936
|
| 3232 | 2024-04-03T22:24:10.720Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 44305
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:37936
req_id = 72359743-65e3-480e-9a81-74232513768d
response_code = 204
uri = /switch-ports
|
| 3233 | 2024-04-03T22:24:10.721Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:34516
|
| 3234 | 2024-04-03T22:24:10.724Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 3008
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:37936
req_id = 1148e1ac-cf8e-4269-a5d8-109678a4ef8f
response_code = 200
uri = /network-bootstore-config
|
| 3235 | 2024-04-03T22:24:10.762Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 41213
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:34516
req_id = 3b49e5fa-0233-41e6-a511-3d77bc378755
response_code = 204
uri = /switch-ports
|
| 3236 | 2024-04-03T22:24:10.763Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 663
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:34516
req_id = 860a8677-994f-4ec0-a4ca-67743a0f8b2e
response_code = 200
uri = /network-bootstore-config
|
| 3237 | 2024-04-03T22:24:11.093Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:64968
|
| 3238 | 2024-04-03T22:24:11.138Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 45307
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:64968
req_id = 6ac8d104-c6f9-4003-80b0-bdfc34357883
response_code = 204
uri = /switch-ports
|
| 3239 | 2024-04-03T22:24:11.138Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 79
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:64968
req_id = 2dc909ea-fa16-49b5-81d6-648374d5844d
response_code = 200
uri = /network-bootstore-config
|
| 3240 | 2024-04-03T22:24:23.381Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 731
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:57652
req_id = 2d738f98-dff8-41bf-8760-1bfb1044d086
response_code = 200
uri = /metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3
|
| 3241 | 2024-04-03T22:24:40.649Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:62626
|
| 3242 | 2024-04-03T22:24:40.649Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 91
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:62626
req_id = abdcc9cd-305e-4ca7-8ecf-3fe33efae8c0
response_code = 200
uri = /baseboard
|
| 3243 | 2024-04-03T22:24:40.767Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:58697
|
| 3244 | 2024-04-03T22:24:40.815Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 47724
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:58697
req_id = 0bb751b3-0a21-4550-81f4-bae04f591f7d
response_code = 204
uri = /switch-ports
|
| 3245 | 2024-04-03T22:24:40.816Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 791
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:58697
req_id = e543cd82-c243-47d4-a21d-33904a0629c7
response_code = 200
uri = /network-bootstore-config
|
| 3246 | 2024-04-03T22:24:40.975Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:54776
|
| 3247 | 2024-04-03T22:24:41.023Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 48063
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:54776
req_id = 47ac46f6-e1b1-46df-9215-2770dd18dab7
response_code = 204
uri = /switch-ports
|
| 3248 | 2024-04-03T22:24:41.024Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 91
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:54776
req_id = d6a54082-cf8e-410f-a294-ae30a9f136fe
response_code = 200
uri = /network-bootstore-config
|
| 3249 | 2024-04-03T22:24:41.368Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:64275
|
| 3250 | 2024-04-03T22:24:41.414Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 45444
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:64275
req_id = 6789f518-eb47-44b0-a164-9193a691f5c0
response_code = 204
uri = /switch-ports
|
| 3251 | 2024-04-03T22:24:41.414Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 96
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:64275
req_id = a7f8e193-d673-4639-9fdb-a4f7c1d3f4e8
response_code = 200
uri = /network-bootstore-config
|
| 3252 | 2024-04-03T22:24:53.381Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 806
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:57652
req_id = cf63e6e8-de74-4c7a-8598-ca7a55ed8b8d
response_code = 200
uri = /metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3
|
| 3253 | 2024-04-03T22:24:57.408Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:801
|
| 3254 | 2024-04-03T22:25:10.684Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:63711
|
| 3255 | 2024-04-03T22:25:10.684Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 71
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:63711
req_id = 6fce4c05-5e27-4fb0-b4a7-bff74f813f9b
response_code = 200
uri = /baseboard
|
| 3256 | 2024-04-03T22:25:10.684Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:42864
|
| 3257 | 2024-04-03T22:25:10.731Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 46793
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:42864
req_id = b3927116-56cb-43ff-959d-46d7bcd73b39
response_code = 204
uri = /switch-ports
|
| 3258 | 2024-04-03T22:25:10.731Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 102
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:42864
req_id = 381bd409-1260-4865-aa32-b88b1a166426
response_code = 200
uri = /network-bootstore-config
|
| 3259 | 2024-04-03T22:25:10.793Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:38125
|
| 3260 | 2024-04-03T22:25:10.841Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 47995
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:38125
req_id = 6b64f940-10bf-4bac-ae96-03ae3bf2b9bc
response_code = 204
uri = /switch-ports
|
| 3261 | 2024-04-03T22:25:10.842Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 92
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:38125
req_id = f236dce6-0649-4591-b79e-82726c852ed9
response_code = 200
uri = /network-bootstore-config
|
| 3262 | 2024-04-03T22:25:11.083Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:62275
|
| 3263 | 2024-04-03T22:25:11.133Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 49808
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:62275
req_id = b755e128-3a70-4b77-a758-525c48c04f53
response_code = 204
uri = /switch-ports
|
| 3264 | 2024-04-03T22:25:11.133Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 98
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:62275
req_id = 4ada6dce-cfc2-4648-bc0e-7b4cd41fda4f
response_code = 200
uri = /network-bootstore-config
|
| 3265 | 2024-04-03T22:25:23.381Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 486
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:57652
req_id = f71b5b3b-1a29-4fe8-8f8b-6db7ddd60d65
response_code = 200
uri = /metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3
|
| 3266 | 2024-04-03T22:25:40.716Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:41724
|
| 3267 | 2024-04-03T22:25:40.717Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 920
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:41724
req_id = 64deca9a-8953-4af5-90a0-e91b7b610659
response_code = 200
uri = /baseboard
|
| 3268 | 2024-04-03T22:25:40.830Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:63867
|
| 3269 | 2024-04-03T22:25:40.877Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 47334
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:63867
req_id = c914c67c-32b4-47fb-80ec-e75a730f74e3
response_code = 204
uri = /switch-ports
|
| 3270 | 2024-04-03T22:25:40.877Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:48807
|
| 3271 | 2024-04-03T22:25:40.881Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 3230
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:63867
req_id = ca5368b3-82ea-418b-bd07-b58eb60563d9
response_code = 200
uri = /network-bootstore-config
|
| 3272 | 2024-04-03T22:25:40.925Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 47385
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:48807
req_id = fe3b48ad-47fe-4930-97ae-21fe32232171
response_code = 204
uri = /switch-ports
|
| 3273 | 2024-04-03T22:25:40.925Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 128
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:48807
req_id = 8a144dd4-48d7-4f7e-90d2-e6e8aeb7b419
response_code = 200
uri = /network-bootstore-config
|
| 3274 | 2024-04-03T22:25:41.402Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:58553
|
| 3275 | 2024-04-03T22:25:41.450Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 47503
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:58553
req_id = 95d844c1-0bdb-4523-b385-5c8a1053a2f4
response_code = 204
uri = /switch-ports
|
| 3276 | 2024-04-03T22:25:41.450Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 84
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:58553
req_id = 9ea01097-4848-4b6a-b597-0054eeb1a451
response_code = 200
uri = /network-bootstore-config
|
| 3277 | 2024-04-03T22:25:53.381Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 641
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:57652
req_id = 2b3f6873-a352-48b0-830a-8077e60aa278
response_code = 200
uri = /metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3
|
| 3278 | 2024-04-03T22:26:10.580Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:34904
|
| 3279 | 2024-04-03T22:26:10.629Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 48658
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:34904
req_id = 2935ae0b-30a5-406d-b9d0-3b353de31e90
response_code = 204
uri = /switch-ports
|
| 3280 | 2024-04-03T22:26:10.629Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 93
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:34904
req_id = 3d12307f-bcb7-4509-abcd-a8a8c76e4012
response_code = 200
uri = /network-bootstore-config
|
| 3281 | 2024-04-03T22:26:10.662Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:42383
|
| 3282 | 2024-04-03T22:26:10.706Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 44023
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:42383
req_id = c8483c38-de57-45c6-a6c9-5e27b51725c8
response_code = 204
uri = /switch-ports
|
| 3283 | 2024-04-03T22:26:10.707Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 81
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:42383
req_id = 88893640-9364-4c83-b0c6-85b58c6ebed0
response_code = 200
uri = /network-bootstore-config
|
| 3284 | 2024-04-03T22:26:10.748Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:39799
|
| 3285 | 2024-04-03T22:26:10.749Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 669
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:39799
req_id = 0866dc38-5d6a-4090-a2d3-9aeb14a5c35f
response_code = 200
uri = /baseboard
|
| 3286 | 2024-04-03T22:26:11.253Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:45987
|
| 3287 | 2024-04-03T22:26:11.300Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 47052
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:45987
req_id = 8790ab62-12de-45fa-baf3-707075275314
response_code = 204
uri = /switch-ports
|
| 3288 | 2024-04-03T22:26:11.300Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 93
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:45987
req_id = f441cd5c-7249-4c03-adbf-be5aa8504d86
response_code = 200
uri = /network-bootstore-config
|
| 3289 | 2024-04-03T22:26:23.381Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 878
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:57652
req_id = 1fe46548-2df2-47a8-af26-2e215677ebaf
response_code = 200
uri = /metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3
|
| 3290 | 2024-04-03T22:26:40.781Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:56231
|
| 3291 | 2024-04-03T22:26:40.782Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 633
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:56231
req_id = de088b84-f0e6-4e4f-b451-87fe8351fbc5
response_code = 200
uri = /baseboard
|
| 3292 | 2024-04-03T22:26:40.844Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:35851
|
| 3293 | 2024-04-03T22:26:40.896Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 51287
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:35851
req_id = 4c9e0c7b-285b-4e70-aa1a-e7a59d0fc343
response_code = 204
uri = /switch-ports
|
| 3294 | 2024-04-03T22:26:40.896Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:52820
|
| 3295 | 2024-04-03T22:26:40.899Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 3210
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:35851
req_id = 93619bd4-21e6-4b75-b282-f565601260af
response_code = 200
uri = /network-bootstore-config
|
| 3296 | 2024-04-03T22:26:40.942Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 46155
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:52820
req_id = c8c96124-43f4-4e6e-858d-df1099b642e4
response_code = 204
uri = /switch-ports
|
| 3297 | 2024-04-03T22:26:40.942Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 111
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:52820
req_id = c5cb3b67-eeb7-4ac7-8f6b-40efdc504719
response_code = 200
uri = /network-bootstore-config
|
| 3298 | 2024-04-03T22:26:41.449Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:52447
|
| 3299 | 2024-04-03T22:26:41.493Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 43962
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:52447
req_id = 28a9365c-5b78-4c48-9bca-b3a2834f5a31
response_code = 204
uri = /switch-ports
|
| 3300 | 2024-04-03T22:26:41.493Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 84
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:52447
req_id = f3b38dc0-c932-4eb3-9b6f-add82859158f
response_code = 200
uri = /network-bootstore-config
|
| 3301 | 2024-04-03T22:26:53.381Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 484
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:57652
req_id = e18f880c-016b-4387-89a0-2c1cb63a1912
response_code = 200
uri = /metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3
|
| 3302 | 2024-04-03T22:27:10.635Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:51368
|
| 3303 | 2024-04-03T22:27:10.682Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 46849
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:51368
req_id = 2e8f29f2-d224-4a3d-bd25-7fbbccd20a90
response_code = 204
uri = /switch-ports
|
| 3304 | 2024-04-03T22:27:10.682Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:50387
|
| 3305 | 2024-04-03T22:27:10.685Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 2634
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:51368
req_id = 49398a5f-e9dd-4864-b521-5c09f1e1ed3a
response_code = 200
uri = /network-bootstore-config
|
| 3306 | 2024-04-03T22:27:10.727Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 44699
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:50387
req_id = 58ccec9f-106b-41c6-8d91-a375f6ecc1a0
response_code = 204
uri = /switch-ports
|
| 3307 | 2024-04-03T22:27:10.727Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 152
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:50387
req_id = a22a9ba4-468d-419a-908c-cf752be6f22a
response_code = 200
uri = /network-bootstore-config
|
| 3308 | 2024-04-03T22:27:10.822Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:64344
|
| 3309 | 2024-04-03T22:27:10.822Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 51
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:64344
req_id = 53fc26c0-eddd-4437-99c1-5cb37513c3f2
response_code = 200
uri = /baseboard
|
| 3310 | 2024-04-03T22:27:11.198Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:44736
|
| 3311 | 2024-04-03T22:27:11.244Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 45912
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:44736
req_id = 95a7d428-eed4-47f4-953d-0ad099d7a7d1
response_code = 204
uri = /switch-ports
|
| 3312 | 2024-04-03T22:27:11.245Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 85
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:44736
req_id = ecc48bf5-aa8b-4b8c-bd86-d4974d17d519
response_code = 200
uri = /network-bootstore-config
|
| 3313 | 2024-04-03T22:27:23.381Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 1066
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:57652
req_id = 1b833611-1067-4e50-8de9-4bf284d3c5f1
response_code = 200
uri = /metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3
|
| 3314 | 2024-04-03T22:27:40.824Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:57673
|
| 3315 | 2024-04-03T22:27:40.875Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 51444
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:57673
req_id = 9c2d14e0-7b2b-42bd-8dbf-385127aeddfb
response_code = 204
uri = /switch-ports
|
| 3316 | 2024-04-03T22:27:40.876Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:43621
|
| 3317 | 2024-04-03T22:27:40.876Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:57068
|
| 3318 | 2024-04-03T22:27:40.876Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 189
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:57673
req_id = de9a13eb-3388-4201-8668-a8984ae7712e
response_code = 200
uri = /network-bootstore-config
|
| 3319 | 2024-04-03T22:27:40.876Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 128
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:57068
req_id = c7d8ac7c-82ac-4f48-a393-554fa4f650e9
response_code = 200
uri = /baseboard
|
| 3320 | 2024-04-03T22:27:40.923Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 46855
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:43621
req_id = e9b8b0c9-bbcf-4788-a372-137156324184
response_code = 204
uri = /switch-ports
|
| 3321 | 2024-04-03T22:27:40.923Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 156
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:43621
req_id = ffda5652-a71e-4cff-a0ad-6f841f8895b3
response_code = 200
uri = /network-bootstore-config
|
| 3322 | 2024-04-03T22:27:41.467Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:35020
|
| 3323 | 2024-04-03T22:27:41.513Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 45595
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:35020
req_id = e0bd9eee-a1d9-4fd3-930c-e2dc5719fc6e
response_code = 204
uri = /switch-ports
|
| 3324 | 2024-04-03T22:27:41.513Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 90
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:35020
req_id = 226b6cd5-dcdc-4698-8f65-4999677faad0
response_code = 200
uri = /network-bootstore-config
|
| 3325 | 2024-04-03T22:27:53.382Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 1819
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:57652
req_id = 6fb3e172-096c-4030-89ec-6dacf11a72dd
response_code = 200
uri = /metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3
|
| 3326 | 2024-04-03T22:28:10.571Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:48192
|
| 3327 | 2024-04-03T22:28:10.619Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 47995
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:48192
req_id = a8636e72-9a18-42b9-a3e6-ca7d2f5035ff
response_code = 204
uri = /switch-ports
|
| 3328 | 2024-04-03T22:28:10.619Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:36751
|
| 3329 | 2024-04-03T22:28:10.623Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 3258
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:48192
req_id = 2a820bdf-d668-4fba-a24e-638c24bbdd92
response_code = 200
uri = /network-bootstore-config
|
| 3330 | 2024-04-03T22:28:10.664Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 44720
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:36751
req_id = d941d68c-138d-4f44-b66e-b7e4c1082a08
response_code = 204
uri = /switch-ports
|
| 3331 | 2024-04-03T22:28:10.665Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 157
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:36751
req_id = 63c3c685-fd42-4e6b-bdea-f1ce63a37440
response_code = 200
uri = /network-bootstore-config
|
| 3332 | 2024-04-03T22:28:10.910Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:63878
|
| 3333 | 2024-04-03T22:28:10.910Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 61
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:63878
req_id = 86a3580d-d0c4-4af3-869d-41fb4667c338
response_code = 200
uri = /baseboard
|
| 3334 | 2024-04-03T22:28:11.114Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:34754
|
| 3335 | 2024-04-03T22:28:11.163Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 49274
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:34754
req_id = 08008fa7-a55d-40a3-a924-bc40aa159e1a
response_code = 204
uri = /switch-ports
|
| 3336 | 2024-04-03T22:28:11.164Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 110
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:34754
req_id = be1d9367-4510-4c69-8ebc-12afebcafc04
response_code = 200
uri = /network-bootstore-config
|
| 3337 | 2024-04-03T22:28:23.381Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 827
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:57652
req_id = f6a80abf-9e1c-4934-8773-21c040709f16
response_code = 200
uri = /metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3
|
| 3338 | 2024-04-03T22:28:40.761Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:60516
|
| 3339 | 2024-04-03T22:28:40.807Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 45982
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:60516
req_id = a4f89186-b006-431c-a7eb-d1ea8b748b9c
response_code = 204
uri = /switch-ports
|
| 3340 | 2024-04-03T22:28:40.807Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:39203
|
| 3341 | 2024-04-03T22:28:40.811Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 3891
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:60516
req_id = 1fa59a15-ac14-4894-a277-e1dbb8aa2657
response_code = 200
uri = /network-bootstore-config
|
| 3342 | 2024-04-03T22:28:40.853Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 45733
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:39203
req_id = da3bdde0-d811-44ef-9a71-f263d80af227
response_code = 204
uri = /switch-ports
|
| 3343 | 2024-04-03T22:28:40.854Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 143
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:39203
req_id = c6019658-18b0-4e71-8760-91e01c7e2694
response_code = 200
uri = /network-bootstore-config
|
| 3344 | 2024-04-03T22:28:40.943Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:51575
|
| 3345 | 2024-04-03T22:28:40.943Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 59
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:51575
req_id = 3a4932be-4414-48ee-a46e-04f41f59475d
response_code = 200
uri = /baseboard
|
| 3346 | 2024-04-03T22:28:41.502Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:49613
|
| 3347 | 2024-04-03T22:28:41.550Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 47540
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:49613
req_id = b2b86b2c-3660-4074-a1bb-4c5212b0a604
response_code = 204
uri = /switch-ports
|
| 3348 | 2024-04-03T22:28:41.550Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 87
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:49613
req_id = a15e7d30-3179-48e3-bdf3-1d1b6c610200
response_code = 200
uri = /network-bootstore-config
|
| 3349 | 2024-04-03T22:28:53.380Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 644
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:57652
req_id = 64dee62b-4e9a-4c17-92b9-f51307f97d2a
response_code = 200
uri = /metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3
|
| 3350 | 2024-04-03T22:29:10.539Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:58887
|
| 3351 | 2024-04-03T22:29:10.585Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 45488
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:58887
req_id = bfc0ea4b-2e0b-48f7-bcbc-e2502cfa4d61
response_code = 204
uri = /switch-ports
|
| 3352 | 2024-04-03T22:29:10.585Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 97
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:58887
req_id = 09baa1cc-f166-4252-a62a-cfb98e42950a
response_code = 200
uri = /network-bootstore-config
|
| 3353 | 2024-04-03T22:29:10.662Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:52341
|
| 3354 | 2024-04-03T22:29:10.702Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 39834
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:52341
req_id = 3da05712-3677-41ba-af60-2ff41996b9cb
response_code = 204
uri = /switch-ports
|
| 3355 | 2024-04-03T22:29:10.703Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 636
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:52341
req_id = 31e5fa16-fd3f-44c8-9c5e-c71d22619fca
response_code = 200
uri = /network-bootstore-config
|
| 3356 | 2024-04-03T22:29:10.975Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:37729
|
| 3357 | 2024-04-03T22:29:10.976Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 86
local_addr = [fdb0:18c0:4d0d:9fb2::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:9fb2::2]:37729
req_id = 4fecb6af-5964-4c26-a850-c70c2b30e1bf
response_code = 200
uri = /baseboard
|
| 3358 | 2024-04-03T22:29:11.150Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:61897
|
| 3359 | 2024-04-03T22:29:11.199Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 48804
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:61897
req_id = 3a7d7b8d-a318-40de-8b98-80126248f797
response_code = 204
uri = /switch-ports
|
| 3360 | 2024-04-03T22:29:11.199Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 91
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:61897
req_id = a505c359-459b-4caf-b702-4c9e49d7d1b7
response_code = 200
uri = /network-bootstore-config
|
| 3361 | 2024-04-03T22:29:13.523Z | INFO | SledAgent (auto-cleanup-task): running automatic periodic zone bundle cleanup
file = sled-agent/src/zone_bundle.rs:300
|
| 3362 | 2024-04-03T22:29:13.523Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 3363 | 2024-04-03T22:29:13.560Z | INFO | SledAgent (auto-cleanup-task): top of bundle cleanup loop
file = sled-agent/src/zone_bundle.rs:289
next_cleanup = Instant { tv_sec: 2215, tv_nsec: 935713439 }
time_to_next_cleanup = 599.99999993s
|
| 3364 | 2024-04-03T22:29:14.642Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:41449
|
| 3365 | 2024-04-03T22:29:14.649Z | INFO | SledAgent (InstanceManager): ensuring instance is registered
file = sled-agent/src/instance_manager.rs:521
hardware = InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname("localshark") }, nics: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(55537))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: "nvme", name: "disk-63813e9560a", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: bdff0a78-722b-4bed-be9c-af91058c9940, read_only_parent: Some(Volume { block_size: 512, id: 012bf74b-f65b-42ae-8fb7-232118cbd3c9, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 49fa2a7b-addb-4be2-a55b-0473230b735f, key: Some("0bHk2w7E82hUPA4bjI1y5IrKa6UfCw77sBlj+f+jH+w="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::19]:19001", "[fd00:1122:3344:101::16]:19001", "[fd00:1122:3344:101::13]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: bdff0a78-722b-4bed-be9c-af91058c9940, key: Some("TM4lmqAZS24OBFez5kFIbuFI2LVHmzaNoaxvdz251hc="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::17]:19001", "[fd00:1122:3344:101::18]:19000", "[fd00:1122:3344:101::14]:19000"] } }] } }], cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAA8p7ODWINYAACns4NYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAPKezg1iDWAAAp7ODWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0NTFkNjE3Yi00ZGQ1LTRjYTYtOWZmZC04Y2E4OTU0ZDBmYTIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTzdUazVXNVk0TklUT0NSNW9XOXQwNDVPNE1veERSMW93VWZyQU5sbC9MciJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") }
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
instance_runtime = InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }
metadata = InstanceMetadata { silo_id: 07d82d8c-d4f6-4493-b115-dd9d73795a93, project_id: 1dfa68bf-3590-4c60-baac-6934da1753ef }
propolis_addr = [fd00:1122:3344:101::1:0]:12400
propolis_id = 58748212-00c9-4674-8645-6c72316d719a
vmm_runtime = VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-04-03T22:29:14.236318Z }
|
| 3366 | 2024-04-03T22:29:14.649Z | INFO | SledAgent (InstanceManager): registering new instance
file = sled-agent/src/instance_manager.rs:555
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3367 | 2024-04-03T22:29:14.650Z | INFO | SledAgent (InstanceManager): initializing new Instance
file = sled-agent/src/instance.rs:938
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
propolis_id = 58748212-00c9-4674-8645-6c72316d719a
state = InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname("localshark") }, nics: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(55537))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: "nvme", name: "disk-63813e9560a", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: bdff0a78-722b-4bed-be9c-af91058c9940, read_only_parent: Some(Volume { block_size: 512, id: 012bf74b-f65b-42ae-8fb7-232118cbd3c9, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 49fa2a7b-addb-4be2-a55b-0473230b735f, key: Some("0bHk2w7E82hUPA4bjI1y5IrKa6UfCw77sBlj+f+jH+w="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::19]:19001", "[fd00:1122:3344:101::16]:19001", "[fd00:1122:3344:101::13]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: bdff0a78-722b-4bed-be9c-af91058c9940, key: Some("TM4lmqAZS24OBFez5kFIbuFI2LVHmzaNoaxvdz251hc="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::17]:19001", "[fd00:1122:3344:101::18]:19000", "[fd00:1122:3344:101::14]:19000"] } }] } }], cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAA8p7ODWINYAACns4NYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAPKezg1iDWAAAp7ODWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0NTFkNjE3Yi00ZGQ1LTRjYTYtOWZmZC04Y2E4OTU0ZDBmYTIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTzdUazVXNVk0TklUT0NSNW9XOXQwNDVPNE1veERSMW93VWZyQU5sbC9MciJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-04-03T22:29:14.236318Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }
|
| 3368 | 2024-04-03T22:29:14.651Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 8505
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:41449
req_id = 4a09e40c-cf1e-4f8e-b6cb-79309a28d247
response_code = 200
uri = /instances/451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3369 | 2024-04-03T22:29:14.931Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:32970
|
| 3370 | 2024-04-03T22:29:14.933Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 1897
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:32970
req_id = 1400f4a3-9ee1-428d-aa5b-be40b70eb55b
response_code = 200
uri = /instances/451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2/state
|
| 3371 | 2024-04-03T22:29:14.957Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte6", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 246, 101, 181]), slot: 0, vni: Vni { inner: 55537 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: "vopte6" } }
|
| 3372 | 2024-04-03T22:29:14.957Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 3373 | 2024-04-03T22:29:14.997Z | INFO | SledAgent (InstanceManager): Configuring new Omicron zone: oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a
file = illumos-utils/src/zone.rs:315
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3374 | 2024-04-03T22:29:15.022Z | INFO | SledAgent (InstanceManager): Installing Omicron zone: oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a
file = illumos-utils/src/zone.rs:354
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3375 | 2024-04-03T22:29:16.729Z | INFO | SledAgent (InstanceManager): Profile for oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="system/illumos/propolis-server">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlInstance0"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::1:0"/>
<propval type="astring" name="listen_port" value="12400"/>
<propval type="astring" name="metric_addr" value="[fd00:1122:3344:101::c]:12221"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3376 | 2024-04-03T22:29:16.729Z | INFO | SledAgent (InstanceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
zone = oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a
|
| 3377 | 2024-04-03T22:29:16.930Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a") failed: Property not found. retry in 31.449555ms
file = illumos-utils/src/svc.rs:36
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
zone = oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a
|
| 3378 | 2024-04-03T22:29:16.971Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a") failed: Property not found. retry in 76.255952ms
file = illumos-utils/src/svc.rs:36
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
zone = oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a
|
| 3379 | 2024-04-03T22:29:17.064Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a") failed: Property not found. retry in 189.800545ms
file = illumos-utils/src/svc.rs:36
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
zone = oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a
|
| 3380 | 2024-04-03T22:29:17.272Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a") failed: Property not found. retry in 277.827713ms
file = illumos-utils/src/svc.rs:36
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
zone = oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a
|
| 3381 | 2024-04-03T22:29:17.566Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a") failed: Property not found. retry in 442.133888ms
file = illumos-utils/src/svc.rs:36
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
zone = oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a
|
| 3382 | 2024-04-03T22:29:18.020Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a") failed: Property not found. retry in 1.458164431s
file = illumos-utils/src/svc.rs:36
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
zone = oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a
|
| 3383 | 2024-04-03T22:29:19.488Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a") failed: Property not found. retry in 1.333310822s
file = illumos-utils/src/svc.rs:36
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
zone = oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a
|
| 3384 | 2024-04-03T22:29:20.833Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a") failed: Property not found. retry in 1.096774018s
file = illumos-utils/src/svc.rs:36
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
zone = oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a
|
| 3385 | 2024-04-03T22:29:21.958Z | INFO | SledAgent (InstanceManager): Started propolis in zone: oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a
file = sled-agent/src/instance.rs:1423
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3386 | 2024-04-03T22:29:21.967Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a") failed: Property not found. retry in 44.672332ms
file = illumos-utils/src/svc.rs:36
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3387 | 2024-04-03T22:29:22.022Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a") failed: Property not found. retry in 101.97598ms
file = illumos-utils/src/svc.rs:36
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3388 | 2024-04-03T22:29:22.134Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a") failed: Property not found. retry in 154.149179ms
file = illumos-utils/src/svc.rs:36
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3389 | 2024-04-03T22:29:22.297Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a") failed: Property not found. retry in 268.823419ms
file = illumos-utils/src/svc.rs:36
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3390 | 2024-04-03T22:29:22.577Z | INFO | SledAgent (InstanceManager): Propolis SMF service is online
file = sled-agent/src/instance.rs:1432
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3391 | 2024-04-03T22:29:22.619Z | INFO | SledAgent (InstanceManager): Propolis HTTP server online
file = sled-agent/src/instance.rs:1446
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3392 | 2024-04-03T22:29:22.620Z | INFO | SledAgent (InstanceManager): Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAA8p7ODWINYAACns4NYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAPKezg1iDWAAAp7ODWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0NTFkNjE3Yi00ZGQ1LTRjYTYtOWZmZC04Y2E4OTU0ZDBmYTIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTzdUazVXNVk0TklUT0NSNW9XOXQwNDVPNE1veERSMW93VWZyQU5sbC9MciJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"), disks: [DiskRequest { device: "nvme", name: "disk-63813e9560a", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: bdff0a78-722b-4bed-be9c-af91058c9940, read_only_parent: Some(Volume { block_size: 512, id: 012bf74b-f65b-42ae-8fb7-232118cbd3c9, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 49fa2a7b-addb-4be2-a55b-0473230b735f, key: Some("0bHk2w7E82hUPA4bjI1y5IrKa6UfCw77sBlj+f+jH+w="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::19]:19001", "[fd00:1122:3344:101::16]:19001", "[fd00:1122:3344:101::13]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: bdff0a78-722b-4bed-be9c-af91058c9940, key: Some("TM4lmqAZS24OBFez5kFIbuFI2LVHmzaNoaxvdz251hc="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::17]:19001", "[fd00:1122:3344:101::18]:19000", "[fd00:1122:3344:101::14]:19000"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: "vopte6", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: "Test description", id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 1dfa68bf-3590-4c60-baac-6934da1753ef, silo_id: 07d82d8c-d4f6-4493-b115-dd9d73795a93 }, name: "localshark", vcpus: 2 } }
file = sled-agent/src/instance.rs:685
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3393 | 2024-04-03T22:29:22.743Z | INFO | SledAgent (InstanceManager): result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })
file = sled-agent/src/instance.rs:687
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3394 | 2024-04-03T22:29:22.743Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-04-03T22:29:22.743581132Z }
file = sled-agent/src/instance.rs:569
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3395 | 2024-04-03T22:29:22.743Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:591
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
new_instance_state = InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }
new_vmm_state = VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-04-03T22:29:22.743581132Z }
propolis_id = 58748212-00c9-4674-8645-6c72316d719a
|
| 3396 | 2024-04-03T22:29:22.744Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }, propolis_id: 58748212-00c9-4674-8645-6c72316d719a, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-04-03T22:29:22.743581132Z } }
|
| 3397 | 2024-04-03T22:29:22.782Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-04-03T22:29:22.782690921Z }
file = sled-agent/src/instance.rs:569
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3398 | 2024-04-03T22:29:22.782Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:591
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
new_instance_state = InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }
new_vmm_state = VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-04-03T22:29:22.782690921Z }
propolis_id = 58748212-00c9-4674-8645-6c72316d719a
|
| 3399 | 2024-04-03T22:29:22.782Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }, propolis_id: 58748212-00c9-4674-8645-6c72316d719a, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-04-03T22:29:22.782690921Z } }
|
| 3400 | 2024-04-03T22:29:23.386Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 6053
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:57652
req_id = f0425f7f-60e6-4993-844a-cedf25b4dc4b
response_code = 200
uri = /metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3
|
| 3401 | 2024-04-03T22:29:31.303Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:59817
|
| 3402 | 2024-04-03T22:29:31.303Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 591
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:59817
req_id = 70a764db-af00-45ea-8b86-2e540a8f7c56
response_code = 200
uri = /instances/451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2/state
|
| 3403 | 2024-04-03T22:29:31.303Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-04-03T22:29:31.303916223Z }
file = sled-agent/src/instance.rs:569
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3404 | 2024-04-03T22:29:31.303Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:591
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
new_instance_state = InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }
new_vmm_state = VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-04-03T22:29:31.303916223Z }
propolis_id = 58748212-00c9-4674-8645-6c72316d719a
|
| 3405 | 2024-04-03T22:29:31.303Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }, propolis_id: 58748212-00c9-4674-8645-6c72316d719a, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-04-03T22:29:31.303916223Z } }
|
| 3406 | 2024-04-03T22:29:31.339Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-04-03T22:29:31.339211555Z }
file = sled-agent/src/instance.rs:569
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3407 | 2024-04-03T22:29:31.339Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:591
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
new_instance_state = InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-03T22:29:31.339211555Z }
new_vmm_state = VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-04-03T22:29:31.339211555Z }
propolis_id = 58748212-00c9-4674-8645-6c72316d719a
|
| 3408 | 2024-04-03T22:29:31.339Z | INFO | SledAgent (InstanceManager): terminating VMM that has exited
file = sled-agent/src/instance.rs:606
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3409 | 2024-04-03T22:29:31.339Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 3410 | 2024-04-03T22:29:31.339Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 3411 | 2024-04-03T22:29:31.339Z | INFO | SledAgent (ZoneBundler): creating zone bundle
context = ZoneBundleContext { storage_dirs: ["/pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/debug/bundle/zone", "/pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/debug/bundle/zone"], cause: TerminatedInstance, extra_log_dirs: ["/pool/ext/7d954909-2e42-44b1-96d9-f536366f6758/crypt/debug", "/pool/ext/0c6b0894-9c77-422a-8363-2705812a672c/crypt/debug", "/pool/ext/af9b6346-5d28-46df-baea-640dad9b76c3/crypt/debug", "/pool/ext/ae4e2e29-ff41-46eb-978b-335ffe5cd37e/crypt/debug", "/pool/ext/4eb310e9-0d98-4a07-821c-54f35a32dc4a/crypt/debug", "/pool/ext/494a3a1c-f913-4e4b-8eec-202ec6c5bfd4/crypt/debug", "/pool/ext/00b9d2d0-59a4-4682-aa18-072c6202c0c8/crypt/debug", "/pool/ext/70a16060-4587-476f-9772-16ea3a5a39aa/crypt/debug", "/pool/ext/761d3524-c333-4332-9563-7d1ceb36709e/crypt/debug"] }
file = sled-agent/src/zone_bundle.rs:437
zone_name = oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a
|
| 3412 | 2024-04-03T22:29:31.968Z | INFO | SledAgent (ZoneBundler): finished zone bundle
file = sled-agent/src/zone_bundle.rs:1236
metadata = ZoneBundleMetadata { id: ZoneBundleId { zone_name: "oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a", bundle_id: 87006862-ded3-4fb6-820d-c5b81449956a }, time_created: 2024-04-03T22:29:31.340275544Z, version: 0, cause: TerminatedInstance }
|
| 3413 | 2024-04-03T22:29:31.968Z | WARN | SledAgent (InstanceManager): Halting and removing zone: oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a
file = sled-agent/src/instance.rs:780
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3414 | 2024-04-03T22:29:33.767Z | INFO | SledAgent (InstanceManager): halt_and_remove_logged: Previous zone state: Running
file = illumos-utils/src/zone.rs:269
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|
| 3415 | 2024-04-03T22:29:33.793Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-03T22:29:31.339211555Z }, propolis_id: 58748212-00c9-4674-8645-6c72316d719a, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-04-03T22:29:33.793117674Z } }
|
| 3416 | 2024-04-03T22:29:33.863Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:59832
|
| 3417 | 2024-04-03T22:29:33.864Z | WARN | SledAgent (PortManager): unset_virtual_nic_host unimplmented
file = illumos-utils/src/opte/port_manager.rs:620
|
| 3418 | 2024-04-03T22:29:33.864Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849
latency_us = 745
local_addr = [fd00:1122:3344:101::1]:12345
method = DELETE
remote_addr = [fd00:1122:3344:101::a]:59832
req_id = b876ac2a-8929-4265-b510-936401b3a508
response_code = 204
uri = /v2p/dd992550-77d6-4fb2-9f40-daf8c9502c8f
|
| 3419 | 2024-04-03T22:29:34.232Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-03T22:29:31.339211555Z }, propolis_id: 58748212-00c9-4674-8645-6c72316d719a, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-04-03T22:29:33.793117674Z } }
|
| 3420 | 2024-04-03T22:29:34.236Z | INFO | SledAgent (InstanceManager): State monitoring task complete
file = sled-agent/src/instance.rs:723
instance_id = 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2
|