| 1 | | [ Jun 20 21:01:28 Enabled. ] |
| 2 | | [ Jun 20 21:01:29 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] |
| 3 | | [ Jun 20 21:01:29 Method "start" exited with status 0. ] |
| 4 | | [ Jun 20 21:01:29 Rereading configuration. ] |
| 5 | | [ Jun 20 21:01:29 No 'refresh' method defined. Treating as :true. ] |
| 6 | | note: configured to log to "/dev/stdout" |
| 7 | 2024-06-20T21:01:29.017Z | INFO | SledAgent: Ensuring contract reaper thread
file = illumos-utils/src/running_zone.rs:163
|
| 8 | 2024-06-20T21:01:29.018Z | INFO | SledAgent (BootstrapAgentStartup): finding links ["net0", "net1"]
file = sled-agent/src/bootstrap/pre_server.rs:255
|
| 9 | 2024-06-20T21:01:29.070Z | INFO | SledAgent (BootstrapAgentStartup): Starting mg-ddm service
addr-objs = [AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]
file = sled-agent/src/bootstrap/pre_server.rs:262
|
| 10 | 2024-06-20T21:01:29.070Z | INFO | SledAgent (BootstrapAgentStartup): Importing mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:42
path = /opt/oxide/mg-ddm/pkg/ddm/manifest.xml
|
| 11 | 2024-06-20T21:01:29.185Z | INFO | SledAgent (BootstrapAgentStartup): Setting mg-ddm interfaces
file = sled-agent/src/bootstrap/maghemite.rs:50
interfaces = ("net0/ll" "net1/ll")
|
| 12 | 2024-06-20T21:01:29.194Z | INFO | SledAgent (BootstrapAgentStartup): Enabling mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:56
|
| 13 | 2024-06-20T21:01:29.202Z | INFO | SledAgent (BootstrapAgentStartup): Ensuring zfs key directory exists
file = sled-agent/src/bootstrap/pre_server.rs:272
path = /var/run/oxide/
|
| 14 | 2024-06-20T21:01:29.805Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }
|
| 15 | 2024-06-20T21:01:29.944Z | INFO | SledAgent: Starting KeyManager
file = sled-agent/src/long_running_tasks.rs:119
|
| 16 | 2024-06-20T21:01:29.944Z | INFO | SledAgent: Starting StorageManager
file = sled-agent/src/long_running_tasks.rs:131
|
| 17 | 2024-06-20T21:01:29.944Z | INFO | SledAgent: Starting StorageMonitor
file = sled-agent/src/long_running_tasks.rs:141
|
| 18 | 2024-06-20T21:01:29.944Z | INFO | SledAgent: Starting HardwareManager
file = sled-agent/src/long_running_tasks.rs:162
nongimlet_observed_disks = []
sled_mode = Scrimlet { asic: SoftNpuZone }
|
| 19 | 2024-06-20T21:01:29.944Z | INFO | SledAgent (HardwareManager): Creating HardwareManager
file = sled-hardware/src/illumos/mod.rs:625
|
| 20 | 2024-06-20T21:01:29.944Z | INFO | SledAgent (DumpSetup-worker): DumpSetup poll loop started.
file = sled-agent/src/dump_setup.rs:506
|
| 21 | 2024-06-20T21:01:29.947Z | INFO | SledAgent (HardwareManager): Generated i86pc baseboard Pc { identifier: "sock", model: "i86pc" }
file = sled-hardware/src/illumos/mod.rs:535
|
| 22 | 2024-06-20T21:01:29.947Z | WARN | SledAgent (HardwareManager): Device is not a Gimlet (i86pc), proceeding with null hardware view
file = sled-hardware/src/illumos/mod.rs:674
|
| 23 | 2024-06-20T21:01:29.947Z | INFO | SledAgent: Starting HardwareMonitor
file = sled-agent/src/long_running_tasks.rs:176
|
| 24 | 2024-06-20T21:01:29.947Z | INFO | SledAgent (HardwareManager): Monitoring for hardware updates
file = sled-hardware/src/illumos/mod.rs:733
|
| 25 | 2024-06-20T21:01:29.947Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/m2_0.vdev
|
| 26 | 2024-06-20T21:01:29.947Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
disks = {}
file = sled-agent/src/hardware_monitor.rs:262
underlay_network_info = None
|
| 27 | 2024-06-20T21:01:29.947Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 28 | 2024-06-20T21:01:29.947Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 29 | 2024-06-20T21:01:29.947Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
path = /scratch/m2_0.vdev
|
| 30 | 2024-06-20T21:01:29.958Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev
file = sled-hardware/src/disk.rs:304
|
| 31 | 2024-06-20T21:01:29.958Z | INFO | SledAgent (StorageResources): Formatting zpool with generated ID
file = sled-hardware/src/disk.rs:315
id = 7a99c490-997c-4274-94ca-e12a3ae10436 (zpool)
|
| 32 | 2024-06-20T21:01:30.056Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 7a99c490-997c-4274-94ca-e12a3ae10436 (zpool), kind: Internal }
|
| 33 | 2024-06-20T21:01:30.262Z | INFO | SledAgent (StorageManager): Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }
file = sled-storage/src/manager.rs:381
|
| 34 | 2024-06-20T21:01:30.262Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/m2_1.vdev
|
| 35 | 2024-06-20T21:01:30.262Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 36 | 2024-06-20T21:01:30.262Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 37 | 2024-06-20T21:01:30.262Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 38 | 2024-06-20T21:01:30.262Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 39 | 2024-06-20T21:01:30.262Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
path = /scratch/m2_1.vdev
|
| 40 | 2024-06-20T21:01:30.271Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev
file = sled-hardware/src/disk.rs:304
|
| 41 | 2024-06-20T21:01:30.271Z | INFO | SledAgent (StorageResources): Formatting zpool with generated ID
file = sled-hardware/src/disk.rs:315
id = 9d01641f-20d1-417f-950c-b1e4c58d23b5 (zpool)
|
| 42 | 2024-06-20T21:01:30.356Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 9d01641f-20d1-417f-950c-b1e4c58d23b5 (zpool), kind: Internal }
|
| 43 | 2024-06-20T21:01:30.558Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_0.vdev
|
| 44 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 7a99c490-997c-4274-94ca-e12a3ae10436 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 9d01641f-20d1-417f-950c-b1e4c58d23b5 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 45 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 46 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 47 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 48 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 49 | 2024-06-20T21:01:30.558Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_1.vdev
|
| 50 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 7a99c490-997c-4274-94ca-e12a3ae10436 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 9d01641f-20d1-417f-950c-b1e4c58d23b5 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 51 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 52 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 53 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 54 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 55 | 2024-06-20T21:01:30.558Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_2.vdev
|
| 56 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 57 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 7a99c490-997c-4274-94ca-e12a3ae10436 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 9d01641f-20d1-417f-950c-b1e4c58d23b5 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 58 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 59 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 60 | 2024-06-20T21:01:30.558Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_3.vdev
|
| 61 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 62 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 7a99c490-997c-4274-94ca-e12a3ae10436 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 9d01641f-20d1-417f-950c-b1e4c58d23b5 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 63 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 64 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 65 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 66 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 67 | 2024-06-20T21:01:30.558Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_4.vdev
|
| 68 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 7a99c490-997c-4274-94ca-e12a3ae10436 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 9d01641f-20d1-417f-950c-b1e4c58d23b5 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 69 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 70 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 71 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 72 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 73 | 2024-06-20T21:01:30.558Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_5.vdev
|
| 74 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 75 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 76 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 7a99c490-997c-4274-94ca-e12a3ae10436 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 9d01641f-20d1-417f-950c-b1e4c58d23b5 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 77 | 2024-06-20T21:01:30.558Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_6.vdev
|
| 78 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 7a99c490-997c-4274-94ca-e12a3ae10436 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 9d01641f-20d1-417f-950c-b1e4c58d23b5 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 79 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 80 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 81 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 82 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 83 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 84 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 85 | 2024-06-20T21:01:30.558Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_7.vdev
|
| 86 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 7a99c490-997c-4274-94ca-e12a3ae10436 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 9d01641f-20d1-417f-950c-b1e4c58d23b5 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 87 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 88 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 89 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 90 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 91 | 2024-06-20T21:01:30.558Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_8.vdev
|
| 92 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 7a99c490-997c-4274-94ca-e12a3ae10436 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 9d01641f-20d1-417f-950c-b1e4c58d23b5 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 93 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 94 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 95 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 96 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 97 | 2024-06-20T21:01:30.558Z | INFO | SledAgent: Waiting for boot disk
file = sled-agent/src/long_running_tasks.rs:91
|
| 98 | 2024-06-20T21:01:30.558Z | INFO | SledAgent: Found boot disk DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
file = sled-agent/src/long_running_tasks.rs:93
|
| 99 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 7a99c490-997c-4274-94ca-e12a3ae10436 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 9d01641f-20d1-417f-950c-b1e4c58d23b5 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 100 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 101 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 102 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 103 | 2024-06-20T21:01:30.558Z | INFO | SledAgent: Starting Bootstore
file = sled-agent/src/long_running_tasks.rs:200
|
| 104 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (bootstore): No ledger in /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 105 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (bootstore): No ledger in /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 106 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (bootstore): No ledger found. Loading Fsm::State as Uninitialized
file = bootstore/src/schemes/v0/storage.rs:83
peer_id = pc-sock-i86pc
|
| 107 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (bootstore): No ledger in /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/cluster/bootstore-network-config.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 108 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (bootstore): No ledger in /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/cluster/bootstore-network-config.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 109 | 2024-06-20T21:01:30.558Z | INFO | SledAgent (bootstore): No ledger found for network config
file = bootstore/src/schemes/v0/storage.rs:146
peer_id = pc-sock-i86pc
|
| 110 | 2024-06-20T21:01:30.558Z | INFO | SledAgent: Starting Bootstore DDMD poller
file = sled-agent/src/long_running_tasks.rs:205
|
| 111 | 2024-06-20T21:01:30.558Z | INFO | SledAgent: Starting ZoneBundler related tasks
file = sled-agent/src/long_running_tasks.rs:220
|
| 112 | 2024-06-20T21:01:30.569Z | INFO | SledAgent (ServiceManager): Creating ServiceManager
file = sled-agent/src/services.rs:711
|
| 113 | 2024-06-20T21:01:30.569Z | INFO | SledAgent (auto-cleanup-task): top of bundle cleanup loop
file = sled-agent/src/zone_bundle.rs:289
next_cleanup = Instant { tv_sec: 1318, tv_nsec: 322247930 }
time_to_next_cleanup = 599.999955809s
|
| 114 | 2024-06-20T21:01:30.569Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 115 | 2024-06-20T21:01:30.569Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3647
|
| 116 | 2024-06-20T21:01:30.569Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/sled-agent-request.json
file = common/src/ledger.rs:188
|
| 117 | 2024-06-20T21:01:30.569Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/sled-agent-request.json
file = common/src/ledger.rs:188
|
| 118 | 2024-06-20T21:01:30.572Z | INFO | SledAgent (dropshot (BootstrapAgent)): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
|
| 119 | 2024-06-20T21:01:30.572Z | INFO | SledAgent (SledAgentSprocketsServer): Started listening
file = sled-agent/src/bootstrap/sprockets_server.rs:44
local_addr = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 120 | 2024-06-20T21:01:30.572Z | INFO | SledAgent (RSS): Injecting RSS configuration: RackInitializeRequest {
trust_quorum_peers: None,
bootstrap_discovery: OnlyOurs,
ntp_servers: [
"0.pool.ntp.org",
],
dns_servers: [
1.1.1.1,
9.9.9.9,
],
internal_services_ip_pool_ranges: [
V4(
Ipv4Range {
first: 10.151.2.164,
last: 10.151.2.173,
},
),
],
external_dns_ips: [
10.151.2.164,
10.151.2.165,
],
external_dns_zone_name: "oxide.test",
external_certificates: "<redacted>",
recovery_silo: RecoverySiloConfig {
silo_name: Name(
"recovery",
),
user_name: UserId(
"recovery",
),
user_password_hash: NewPasswordHash(
PasswordHashString {
string: "$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY",
encoding: B64,
},
),
},
rack_network_config: RackNetworkConfigV2 {
rack_subnet: Ipv6Net {
addr: fd00:1122:3344:100::,
width: 56,
},
infra_ip_first: 10.151.2.174,
infra_ip_last: 10.151.2.174,
ports: [
PortConfigV2 {
routes: [
RouteConfig {
destination: V4(
Ipv4Net {
addr: 0.0.0.0,
width: 0,
},
),
nexthop: 10.151.2.100,
vlan_id: None,
},
],
addresses: [
UplinkAddressConfig {
address: V4(
Ipv4Net {
addr: 10.151.2.174,
width: 24,
},
),
vlan_id: None,
},
],
switch: Switch0,
port: "qsfp0",
uplink_port_speed: Speed40G,
uplink_port_fec: None,
bgp_peers: [],
autoneg: false,
},
],
bgp: [],
bfd: [],
},
allowed_source_ips: Any,
}
file = sled-agent/src/rack_setup/service.rs:1011
|
| 121 | 2024-06-20T21:01:30.572Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
| 122 | 2024-06-20T21:01:30.572Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 123 | 2024-06-20T21:01:30.572Z | INFO | SledAgent (RSS): No ledger in /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/rss-plan-completed.marker
file = common/src/ledger.rs:188
|
| 124 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (RSS): No ledger in /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/rss-plan-completed.marker
file = common/src/ledger.rs:188
|
| 125 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (RSS): RSS configuration has not been fully applied yet
file = sled-agent/src/rack_setup/service.rs:1071
|
| 126 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 127 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (RSS): No ledger in /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/rss-sled-plan.json
file = common/src/ledger.rs:188
|
| 128 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (RSS): No ledger in /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/rss-sled-plan.json
file = common/src/ledger.rs:188
|
| 129 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (RSS): Creating new allocation plan
file = sled-agent/src/rack_setup/service.rs:1112
|
| 130 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (RSS): Serializing plan
file = sled-agent/src/rack_setup/plan/sled.rs:156
|
| 131 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (RSS): Creating plan for the sled at fdb0:18c0:4d0d:a02a::1
file = sled-agent/src/rack_setup/plan/sled.rs:129
|
| 132 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 133 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/.rss-sled-plan.json.tmp
file = common/src/ledger.rs:199
|
| 134 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/.rss-sled-plan.json.tmp
file = common/src/ledger.rs:199
|
| 135 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (RSS): Sled plan written to storage
file = sled-agent/src/rack_setup/plan/sled.rs:176
|
| 136 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (RSS): Writing Rack Network Configuration to bootstore
file = sled-agent/src/rack_setup/service.rs:1145
|
| 137 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 1, current_generation: 0
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
| 138 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 139 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 140 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 1
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-sock-i86pc
|
| 141 | 2024-06-20T21:01:30.573Z | INFO | SledAgent (BootstrapAgentRssHandler): Received initialization request from RSS
file = sled-agent/src/bootstrap/rss_handle.rs:217
request = StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: ebe60f78-650f-482d-ab07-6cce79648e59, rack_id: 0f15006d-98e4-48dc-8b6c-3e866ca1aace, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 142 | 2024-06-20T21:01:30.574Z | INFO | SledAgent (SledAgentSprocketsServer): Accepted connection
file = sled-agent/src/bootstrap/sprockets_server.rs:66
remote_addr = [fdb0:18c0:4d0d:a02a::1]:38047
|
| 143 | 2024-06-20T21:01:30.574Z | INFO | SledAgent (SledAgentMain): Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: ebe60f78-650f-482d-ab07-6cce79648e59, rack_id: 0f15006d-98e4-48dc-8b6c-3e866ca1aace, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }
file = sled-agent/src/bootstrap/server.rs:360
|
| 144 | 2024-06-20T21:01:30.574Z | INFO | SledAgent (SledAgentMain): KeyManager: using hardcoded secret retriever
file = sled-agent/src/bootstrap/server.rs:375
|
| 145 | 2024-06-20T21:01:30.574Z | INFO | SledAgent: setting up sled agent server
file = sled-agent/src/server.rs:44
|
| 146 | 2024-06-20T21:01:30.574Z | INFO | SledAgent (StorageManager): Received KeyManagerReady
file = sled-storage/src/manager.rs:381
|
| 147 | 2024-06-20T21:01:30.574Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
| 148 | 2024-06-20T21:01:30.574Z | INFO | SledAgent (StorageManager): Loading storage config
file = sled-storage/src/manager.rs:530
|
| 149 | 2024-06-20T21:01:30.574Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }
|
| 150 | 2024-06-20T21:01:30.574Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = load_ledger
|
| 151 | 2024-06-20T21:01:30.574Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = load_ledger
|
| 152 | 2024-06-20T21:01:30.574Z | INFO | SledAgent (StorageManager): No ledger of physical disks exists
file = sled-storage/src/manager.rs:519
|
| 153 | 2024-06-20T21:01:30.574Z | INFO | SledAgent (StorageManager): KeyManager ready, but no ledger detected
file = sled-storage/src/manager.rs:547
|
| 154 | 2024-06-20T21:01:30.576Z | INFO | SledAgent (ServiceManager): Enabling switch zone (new)
file = sled-agent/src/services.rs:3879
|
| 155 | 2024-06-20T21:01:30.584Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 156 | 2024-06-20T21:01:30.593Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 157 | 2024-06-20T21:01:30.603Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 158 | 2024-06-20T21:01:30.613Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 159 | 2024-06-20T21:01:30.622Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 160 | 2024-06-20T21:01:30.630Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 161 | 2024-06-20T21:01:30.639Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 162 | 2024-06-20T21:01:30.647Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 163 | 2024-06-20T21:01:30.657Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 164 | 2024-06-20T21:01:30.657Z | INFO | SledAgent (StorageManager): No disks to be automatically managed
file = sled-storage/src/manager.rs:558
|
| 165 | 2024-06-20T21:01:30.657Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 166 | 2024-06-20T21:01:30.657Z | INFO | SledAgent (StorageManager): automatically managing disks
file = sled-storage/src/manager.rs:370
|
| 167 | 2024-06-20T21:01:30.657Z | INFO | SledAgent (StorageResources): Synchronizing disk managment
file = sled-storage/src/resources.rs:339
|
| 168 | 2024-06-20T21:01:30.657Z | INFO | SledAgent (StorageManager): Successfully synchronized disks without error
file = sled-storage/src/manager.rs:449
result = DisksManagementResult { status: [] }
|
| 169 | 2024-06-20T21:01:30.669Z | INFO | SledAgent: SledAgent::new(..) starting
file = sled-agent/src/sled_agent.rs:354
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 170 | 2024-06-20T21:01:30.669Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 171 | 2024-06-20T21:01:30.669Z | INFO | SledAgent: Requested swap device of size 64 GiB
file = sled-agent/src/sled_agent.rs:366
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 172 | 2024-06-20T21:01:30.669Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 173 | 2024-06-20T21:01:30.675Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 68.573911ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3cf23413-43d2-4103-b8d8-98939dc9d508", "content-length": "124", "date": "Thu, 20 Jun 2024 21:01:29 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3cf23413-43d2-4103-b8d8-98939dc9d508" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 174 | 2024-06-20T21:01:30.679Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:315
|
| 175 | 2024-06-20T21:01:30.682Z | INFO | SledAgent: attempting to create encrypted zvol: name="oxi_7a99c490-997c-4274-94ca-e12a3ae10436/swap", size_gb=64
file = sled-agent/src/swap_device.rs:153
|
| 176 | 2024-06-20T21:01:30.697Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:354
|
| 177 | 2024-06-20T21:01:30.705Z | INFO | SledAgent: successfully created encrypted zvol: name="oxi_7a99c490-997c-4274-94ca-e12a3ae10436/swap", size_gb=64
file = sled-agent/src/swap_device.rs:249
|
| 178 | 2024-06-20T21:01:30.705Z | INFO | SledAgent: adding swap device: swapname="/dev/zvol/dsk/oxi_7a99c490-997c-4274-94ca-e12a3ae10436/swap"
file = sled-agent/src/swap_device.rs:96
|
| 179 | 2024-06-20T21:01:30.709Z | INFO | SledAgent: Mounting backing filesystems
file = sled-agent/src/sled_agent.rs:381
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 180 | 2024-06-20T21:01:30.709Z | INFO | SledAgent (BackingFs): Processing fmd
file = sled-agent/src/backing_fs.rs:126
|
| 181 | 2024-06-20T21:01:30.709Z | INFO | SledAgent (BackingFs): Ensuring dataset oxi_7a99c490-997c-4274-94ca-e12a3ae10436/backing/fmd
file = sled-agent/src/backing_fs.rs:136
|
| 182 | 2024-06-20T21:01:30.744Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 183 | 2024-06-20T21:01:30.746Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 91.384748ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "ce9f058f-9f55-4110-b954-9e47c9717d7e", "content-length": "124", "date": "Thu, 20 Jun 2024 21:01:29 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "ce9f058f-9f55-4110-b954-9e47c9717d7e" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 184 | 2024-06-20T21:01:30.780Z | INFO | SledAgent (BackingFs): Stopping service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:170
|
| 185 | 2024-06-20T21:01:30.838Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 186 | 2024-06-20T21:01:30.839Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 130.339231ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "af7e4b6a-1dbe-41ee-8e31-7b0e2390fe2a", "content-length": "124", "date": "Thu, 20 Jun 2024 21:01:30 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "af7e4b6a-1dbe-41ee-8e31-7b0e2390fe2a" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 187 | 2024-06-20T21:01:30.971Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 188 | 2024-06-20T21:01:30.971Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 111.294277ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "95836d9c-602c-41ec-a92d-9098996f424d", "content-length": "124", "date": "Thu, 20 Jun 2024 21:01:30 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "95836d9c-602c-41ec-a92d-9098996f424d" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 189 | 2024-06-20T21:01:30.999Z | INFO | SledAgent (BackingFs): Mounting oxi_7a99c490-997c-4274-94ca-e12a3ae10436/backing/fmd on /var/fm/fmd
file = sled-agent/src/backing_fs.rs:178
|
| 190 | 2024-06-20T21:01:31.011Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/rsrc
file = sled-agent/src/backing_fs.rs:186
|
| 191 | 2024-06-20T21:01:31.011Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/ckpt
file = sled-agent/src/backing_fs.rs:186
|
| 192 | 2024-06-20T21:01:31.011Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/xprt
file = sled-agent/src/backing_fs.rs:186
|
| 193 | 2024-06-20T21:01:31.011Z | INFO | SledAgent (BackingFs): Starting service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:192
|
| 194 | 2024-06-20T21:01:31.083Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 195 | 2024-06-20T21:01:31.084Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 188.361334ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f3aa672d-9185-4fb1-ad02-15c2fa96b58e", "content-length": "124", "date": "Thu, 20 Jun 2024 21:01:30 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f3aa672d-9185-4fb1-ad02-15c2fa96b58e" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 196 | 2024-06-20T21:01:31.273Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 197 | 2024-06-20T21:01:31.274Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 242.744905ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "7a75d975-7ae3-4b9e-96aa-68d568a84185", "content-length": "124", "date": "Thu, 20 Jun 2024 21:01:30 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "7a75d975-7ae3-4b9e-96aa-68d568a84185" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 198 | 2024-06-20T21:01:31.518Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 199 | 2024-06-20T21:01:31.519Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 343.61283ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "85629989-5f39-46fc-92bd-bbec3598fa61", "content-length": "124", "date": "Thu, 20 Jun 2024 21:01:30 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "85629989-5f39-46fc-92bd-bbec3598fa61" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 200 | 2024-06-20T21:01:31.704Z | INFO | SledAgent: using '[AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]' as data links for xde driver
file = illumos-utils/src/opte/illumos.rs:90
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 201 | 2024-06-20T21:01:31.705Z | INFO | SledAgent (dropshot): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196
local_addr = [fd00:1122:3344:101::1]:57395
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 202 | 2024-06-20T21:01:31.705Z | INFO | SledAgent (internal-dns-resolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 203 | 2024-06-20T21:01:31.705Z | INFO | SledAgent (producer-server): starting oximeter metric producer server
address = [fd00:1122:3344:101::1]:57395
file = oximeter/producer/src/lib.rs:276
interval = 30s
producer_id = ebe60f78-650f-482d-ab07-6cce79648e59
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 204 | 2024-06-20T21:01:31.706Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 318.821652ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:384
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 205 | 2024-06-20T21:01:31.751Z | INFO | SledAgent (VmmReservoirManager): 50% of 102952026112 physical ram = 51476013056 bytes)
file = sled-agent/src/vmm_reservoir.rs:267
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 206 | 2024-06-20T21:01:31.751Z | INFO | SledAgent (VmmReservoirManager): Setting reservoir size to 49090 MiB bytes
file = sled-agent/src/vmm_reservoir.rs:275
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 207 | 2024-06-20T21:01:31.751Z | INFO | SledAgent (ServiceManager): sled agent started
file = sled-agent/src/services.rs:993
underlay_address = fd00:1122:3344:101::1
|
| 208 | 2024-06-20T21:01:31.751Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
| 209 | 2024-06-20T21:01:31.752Z | INFO | SledAgent (dropshot (SledAgent)): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196
local_addr = [fd00:1122:3344:101::1]:12345
|
| 210 | 2024-06-20T21:01:31.752Z | INFO | SledAgent (SledAgentMain): Sled Agent loaded; recording configuration
file = sled-agent/src/bootstrap/server.rs:430
|
| 211 | 2024-06-20T21:01:31.752Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 212 | 2024-06-20T21:01:31.752Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:199
|
| 213 | 2024-06-20T21:01:31.752Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:199
|
| 214 | 2024-06-20T21:01:31.752Z | INFO | SledAgent (HardwareMonitor): Sled Agent Started
file = sled-agent/src/hardware_monitor.rs:138
|
| 215 | 2024-06-20T21:01:31.752Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
disks = {}
file = sled-agent/src/hardware_monitor.rs:262
underlay_network_info = Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: "qsfp0", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))
|
| 216 | 2024-06-20T21:01:31.752Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3647
|
| 217 | 2024-06-20T21:01:31.752Z | INFO | SledAgent (SledAgentSprocketsServer): Connection closed
file = sled-agent/src/bootstrap/sprockets_server.rs:73
remote_addr = [fdb0:18c0:4d0d:a02a::1]:38047
|
| 218 | 2024-06-20T21:01:31.752Z | INFO | SledAgent (BootstrapAgentRssHandler): Peer agent initialized
file = sled-agent/src/bootstrap/rss_handle.rs:108
peer = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 219 | 2024-06-20T21:01:31.752Z | INFO | SledAgent (BootstrapAgentRssHandler): Initialized sled agent
file = sled-agent/src/bootstrap/rss_handle.rs:232
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 220 | 2024-06-20T21:01:31.942Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 221 | 2024-06-20T21:01:31.942Z | INFO | SledAgent (RSS): No ledger in /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/rss-service-plan-v3.json
file = common/src/ledger.rs:188
|
| 222 | 2024-06-20T21:01:31.943Z | INFO | SledAgent (RSS): No ledger in /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/rss-service-plan-v3.json
file = common/src/ledger.rs:188
|
| 223 | 2024-06-20T21:01:31.943Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 224 | 2024-06-20T21:01:31.943Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 225 | 2024-06-20T21:01:31.943Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 226 | 2024-06-20T21:01:31.950Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 227 | 2024-06-20T21:01:31.978Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:33955
|
| 228 | 2024-06-20T21:01:31.978Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 229 | 2024-06-20T21:01:31.978Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 180
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:33955
req_id = 9a9af9fc-542f-40d8-9330-7cef5f25bc94
response_code = 200
uri = /inventory
|
| 230 | 2024-06-20T21:01:32.009Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:62229
|
| 231 | 2024-06-20T21:01:32.010Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 45
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:62229
req_id = 5a506855-9425-45f5-9ec7-91dce25ed969
response_code = 200
uri = /sled-role
|
| 232 | 2024-06-20T21:01:32.011Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 233 | 2024-06-20T21:01:32.011Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/.rss-service-plan-v3.json.tmp
file = common/src/ledger.rs:199
|
| 234 | 2024-06-20T21:01:32.011Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/.rss-service-plan-v3.json.tmp
file = common/src/ledger.rs:199
|
| 235 | 2024-06-20T21:01:32.012Z | INFO | SledAgent (RSS): Service plan written to storage
file = sled-agent/src/rack_setup/plan/service.rs:812
|
| 236 | 2024-06-20T21:01:32.026Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 372.924458ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:384
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 237 | 2024-06-20T21:01:32.042Z | INFO | SledAgent (RSS): attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 030fdd9b-c9d8-4714-80f3-a1d056ffa95b, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, pool_id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool) }, OmicronPhysicalDiskConfig { id: 06c73f1a-62f3-4647-a302-71ce7ad5dc38, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, pool_id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool) }, OmicronPhysicalDiskConfig { id: bb52e57f-dd7a-4400-9511-bb075cceba78, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, pool_id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool) }, OmicronPhysicalDiskConfig { id: d806b53b-d088-45e7-97b4-291c1065fbc8, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, pool_id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool) }, OmicronPhysicalDiskConfig { id: e7014d16-fee3-413e-a186-8e9146dc76d3, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, pool_id: 63627367-eb03-4016-8c1d-73768441f612 (zpool) }, OmicronPhysicalDiskConfig { id: 2d142903-9002-45c9-9f48-5e2103f18d56, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, pool_id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool) }, OmicronPhysicalDiskConfig { id: 4f5dd250-2c98-45de-8214-27f215685f19, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, pool_id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool) }, OmicronPhysicalDiskConfig { id: 7b859164-d9d5-4d37-ae9f-4300dc4c8300, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, pool_id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool) }, OmicronPhysicalDiskConfig { id: 44f36d70-b78e-4ca2-a7f9-3b610d259c0a, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, pool_id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool) }], generation: Generation(1) }
file = sled-agent/src/rack_setup/service.rs:348
sled_address = [fd00:1122:3344:101::1]:12345
|
| 238 | 2024-06-20T21:01:32.042Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:49860
|
| 239 | 2024-06-20T21:01:32.043Z | INFO | SledAgent (StorageManager): Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, id: 030fdd9b-c9d8-4714-80f3-a1d056ffa95b, pool_id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, id: 06c73f1a-62f3-4647-a302-71ce7ad5dc38, pool_id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, id: bb52e57f-dd7a-4400-9511-bb075cceba78, pool_id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, id: d806b53b-d088-45e7-97b4-291c1065fbc8, pool_id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, id: e7014d16-fee3-413e-a186-8e9146dc76d3, pool_id: 63627367-eb03-4016-8c1d-73768441f612 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, id: 2d142903-9002-45c9-9f48-5e2103f18d56, pool_id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, id: 4f5dd250-2c98-45de-8214-27f215685f19, pool_id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, id: 7b859164-d9d5-4d37-ae9f-4300dc4c8300, pool_id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, id: 44f36d70-b78e-4ca2-a7f9-3b610d259c0a, pool_id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool) }] }, tx: ... }
file = sled-storage/src/manager.rs:381
|
| 240 | 2024-06-20T21:01:32.043Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = omicron_physical_disks_ensure
|
| 241 | 2024-06-20T21:01:32.043Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = omicron_physical_disks_ensure
|
| 242 | 2024-06-20T21:01:32.043Z | INFO | SledAgent (StorageManager): No previously-stored 'requested disks', creating new ledger
file = sled-storage/src/manager.rs:681
request = omicron_physical_disks_ensure
|
| 243 | 2024-06-20T21:01:32.043Z | INFO | SledAgent (StorageResources): Synchronizing disk managment
file = sled-storage/src/resources.rs:339
|
| 244 | 2024-06-20T21:01:32.043Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:379
|
| 245 | 2024-06-20T21:01:32.043Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 246 | 2024-06-20T21:01:32.043Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
path = /scratch/u2_0.vdev
|
| 247 | 2024-06-20T21:01:32.153Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev
file = sled-hardware/src/disk.rs:304
|
| 248 | 2024-06-20T21:01:32.153Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = cc5f6a04-2111-4941-923c-8867d737d5fc (zpool)
|
| 249 | 2024-06-20T21:01:32.543Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 1.226020668s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:384
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 250 | 2024-06-20T21:01:32.779Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 251 | 2024-06-20T21:01:33.217Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }
|
| 252 | 2024-06-20T21:01:33.392Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:332
|
| 253 | 2024-06-20T21:01:33.392Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 254 | 2024-06-20T21:01:33.392Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 255 | 2024-06-20T21:01:33.392Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 256 | 2024-06-20T21:01:33.392Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 257 | 2024-06-20T21:01:33.392Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 258 | 2024-06-20T21:01:33.392Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 259 | 2024-06-20T21:01:33.794Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 1.040013983s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:384
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 260 | 2024-06-20T21:01:33.794Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 261 | 2024-06-20T21:01:33.985Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_switch
|
| 262 | 2024-06-20T21:01:34.181Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 263 | 2024-06-20T21:01:34.593Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 264 | 2024-06-20T21:01:34.594Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 265 | 2024-06-20T21:01:34.777Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 64.971575ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 266 | 2024-06-20T21:01:35.142Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 3.0993043s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:384
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 267 | 2024-06-20T21:01:35.142Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 268 | 2024-06-20T21:01:35.325Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 130.333224ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 269 | 2024-06-20T21:01:35.757Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 132.292006ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 270 | 2024-06-20T21:01:36.314Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 268.917573ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 271 | 2024-06-20T21:01:36.317Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 272 | 2024-06-20T21:01:36.317Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 273 | 2024-06-20T21:01:36.982Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 677.810625ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 274 | 2024-06-20T21:01:37.695Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 275 | 2024-06-20T21:01:37.695Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_cc5f6a04-2111-4941-923c-8867d737d5fc
file = sled-storage/src/dataset.rs:480
|
| 276 | 2024-06-20T21:01:37.918Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 1.250331083s
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 277 | 2024-06-20T21:01:37.927Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:477
|
| 278 | 2024-06-20T21:01:37.927Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:394
|
| 279 | 2024-06-20T21:01:37.927Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:379
|
| 280 | 2024-06-20T21:01:37.927Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 281 | 2024-06-20T21:01:37.927Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
path = /scratch/u2_1.vdev
|
| 282 | 2024-06-20T21:01:38.201Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev
file = sled-hardware/src/disk.rs:304
|
| 283 | 2024-06-20T21:01:38.201Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 662892f9-e563-4653-bedf-02b82422ec9a (zpool)
|
| 284 | 2024-06-20T21:01:39.419Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }
|
| 285 | 2024-06-20T21:01:39.702Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:332
|
| 286 | 2024-06-20T21:01:39.702Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 287 | 2024-06-20T21:01:39.702Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 288 | 2024-06-20T21:01:39.702Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 289 | 2024-06-20T21:01:39.702Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 290 | 2024-06-20T21:01:39.702Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 291 | 2024-06-20T21:01:39.702Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 292 | 2024-06-20T21:01:40.194Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 4.381189277s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:384
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 293 | 2024-06-20T21:01:40.194Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 294 | 2024-06-20T21:01:40.194Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 295 | 2024-06-20T21:01:41.170Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 296 | 2024-06-20T21:01:41.669Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 297 | 2024-06-20T21:01:41.669Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_662892f9-e563-4653-bedf-02b82422ec9a/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 298 | 2024-06-20T21:01:42.150Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 299 | 2024-06-20T21:01:42.360Z | INFO | SledAgent (VmmReservoirManager): Finished setting reservoir size to 49090 MiB bytes
file = sled-agent/src/vmm_reservoir.rs:279
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 300 | 2024-06-20T21:01:42.471Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_662892f9-e563-4653-bedf-02b82422ec9a
file = sled-storage/src/dataset.rs:480
|
| 301 | 2024-06-20T21:01:42.479Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:477
|
| 302 | 2024-06-20T21:01:42.479Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:394
|
| 303 | 2024-06-20T21:01:42.479Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:379
|
| 304 | 2024-06-20T21:01:42.479Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 305 | 2024-06-20T21:01:42.479Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
path = /scratch/u2_2.vdev
|
| 306 | 2024-06-20T21:01:42.489Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev
file = sled-hardware/src/disk.rs:304
|
| 307 | 2024-06-20T21:01:42.489Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool)
|
| 308 | 2024-06-20T21:01:42.584Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }
|
| 309 | 2024-06-20T21:01:42.592Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/dataset.rs:332
|
| 310 | 2024-06-20T21:01:42.592Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 311 | 2024-06-20T21:01:42.592Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 312 | 2024-06-20T21:01:42.592Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 313 | 2024-06-20T21:01:42.592Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 314 | 2024-06-20T21:01:42.592Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 315 | 2024-06-20T21:01:42.592Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 316 | 2024-06-20T21:01:42.628Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 317 | 2024-06-20T21:01:42.636Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_f32a5037-29da-4375-a8f1-8e9a8b5d3778/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 318 | 2024-06-20T21:01:42.671Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 319 | 2024-06-20T21:01:42.758Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_f32a5037-29da-4375-a8f1-8e9a8b5d3778
file = sled-storage/src/dataset.rs:480
|
| 320 | 2024-06-20T21:01:42.766Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:477
|
| 321 | 2024-06-20T21:01:42.766Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:394
|
| 322 | 2024-06-20T21:01:42.766Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:379
|
| 323 | 2024-06-20T21:01:42.766Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 324 | 2024-06-20T21:01:42.766Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
path = /scratch/u2_3.vdev
|
| 325 | 2024-06-20T21:01:42.776Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev
file = sled-hardware/src/disk.rs:304
|
| 326 | 2024-06-20T21:01:42.776Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 339b70a8-c294-4120-9bd3-76df19004c3b (zpool)
|
| 327 | 2024-06-20T21:01:42.870Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }
|
| 328 | 2024-06-20T21:01:42.877Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/dataset.rs:332
|
| 329 | 2024-06-20T21:01:42.877Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 330 | 2024-06-20T21:01:42.877Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 331 | 2024-06-20T21:01:42.877Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 332 | 2024-06-20T21:01:42.877Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 333 | 2024-06-20T21:01:42.878Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 334 | 2024-06-20T21:01:42.878Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 335 | 2024-06-20T21:01:42.912Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 336 | 2024-06-20T21:01:42.919Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_339b70a8-c294-4120-9bd3-76df19004c3b/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 337 | 2024-06-20T21:01:43.038Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_339b70a8-c294-4120-9bd3-76df19004c3b
file = sled-storage/src/dataset.rs:480
|
| 338 | 2024-06-20T21:01:43.045Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:477
|
| 339 | 2024-06-20T21:01:43.045Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:394
|
| 340 | 2024-06-20T21:01:43.045Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:379
|
| 341 | 2024-06-20T21:01:43.045Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 342 | 2024-06-20T21:01:43.045Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
path = /scratch/u2_4.vdev
|
| 343 | 2024-06-20T21:01:43.054Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev
file = sled-hardware/src/disk.rs:304
|
| 344 | 2024-06-20T21:01:43.054Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 63627367-eb03-4016-8c1d-73768441f612 (zpool)
|
| 345 | 2024-06-20T21:01:43.147Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }
|
| 346 | 2024-06-20T21:01:43.155Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/dataset.rs:332
|
| 347 | 2024-06-20T21:01:43.155Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 348 | 2024-06-20T21:01:43.155Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 349 | 2024-06-20T21:01:43.155Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 350 | 2024-06-20T21:01:43.155Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 351 | 2024-06-20T21:01:43.155Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 352 | 2024-06-20T21:01:43.155Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 353 | 2024-06-20T21:01:43.189Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 354 | 2024-06-20T21:01:43.197Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_63627367-eb03-4016-8c1d-73768441f612/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 355 | 2024-06-20T21:01:43.320Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_63627367-eb03-4016-8c1d-73768441f612
file = sled-storage/src/dataset.rs:480
|
| 356 | 2024-06-20T21:01:43.327Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:477
|
| 357 | 2024-06-20T21:01:43.327Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:394
|
| 358 | 2024-06-20T21:01:43.327Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:379
|
| 359 | 2024-06-20T21:01:43.327Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 360 | 2024-06-20T21:01:43.327Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
path = /scratch/u2_5.vdev
|
| 361 | 2024-06-20T21:01:43.337Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev
file = sled-hardware/src/disk.rs:304
|
| 362 | 2024-06-20T21:01:43.337Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 2e39fd13-5176-4455-b633-7660296b7576 (zpool)
|
| 363 | 2024-06-20T21:01:43.425Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }
|
| 364 | 2024-06-20T21:01:43.433Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/dataset.rs:332
|
| 365 | 2024-06-20T21:01:43.433Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 366 | 2024-06-20T21:01:43.433Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 367 | 2024-06-20T21:01:43.433Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 368 | 2024-06-20T21:01:43.433Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 369 | 2024-06-20T21:01:43.433Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 370 | 2024-06-20T21:01:43.433Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 371 | 2024-06-20T21:01:43.467Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 372 | 2024-06-20T21:01:43.475Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_2e39fd13-5176-4455-b633-7660296b7576/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 373 | 2024-06-20T21:01:43.597Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_2e39fd13-5176-4455-b633-7660296b7576
file = sled-storage/src/dataset.rs:480
|
| 374 | 2024-06-20T21:01:43.605Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:477
|
| 375 | 2024-06-20T21:01:43.605Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:394
|
| 376 | 2024-06-20T21:01:43.605Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:379
|
| 377 | 2024-06-20T21:01:43.605Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 378 | 2024-06-20T21:01:43.605Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
path = /scratch/u2_6.vdev
|
| 379 | 2024-06-20T21:01:43.616Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev
file = sled-hardware/src/disk.rs:304
|
| 380 | 2024-06-20T21:01:43.616Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool)
|
| 381 | 2024-06-20T21:01:43.675Z | INFO | SledAgent (ServiceManager): Ensuring tfportrear0_0/ll exists in zone
file = sled-agent/src/services.rs:2333
|
| 382 | 2024-06-20T21:01:43.716Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }
|
| 383 | 2024-06-20T21:01:43.724Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/dataset.rs:332
|
| 384 | 2024-06-20T21:01:43.724Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 385 | 2024-06-20T21:01:43.724Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 386 | 2024-06-20T21:01:43.724Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 387 | 2024-06-20T21:01:43.724Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 388 | 2024-06-20T21:01:43.724Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 389 | 2024-06-20T21:01:43.724Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 390 | 2024-06-20T21:01:43.728Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 391 | 2024-06-20T21:01:43.761Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 392 | 2024-06-20T21:01:43.768Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_9ff415f8-ea2c-44b4-b9a7-2a7cf086871c/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 393 | 2024-06-20T21:01:43.892Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_9ff415f8-ea2c-44b4-b9a7-2a7cf086871c
file = sled-storage/src/dataset.rs:480
|
| 394 | 2024-06-20T21:01:43.900Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:477
|
| 395 | 2024-06-20T21:01:43.900Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:394
|
| 396 | 2024-06-20T21:01:43.900Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:379
|
| 397 | 2024-06-20T21:01:43.900Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 398 | 2024-06-20T21:01:43.900Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
path = /scratch/u2_7.vdev
|
| 399 | 2024-06-20T21:01:43.910Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev
file = sled-hardware/src/disk.rs:304
|
| 400 | 2024-06-20T21:01:43.910Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool)
|
| 401 | 2024-06-20T21:01:44.005Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }
|
| 402 | 2024-06-20T21:01:44.013Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/dataset.rs:332
|
| 403 | 2024-06-20T21:01:44.013Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 404 | 2024-06-20T21:01:44.013Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 405 | 2024-06-20T21:01:44.013Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 406 | 2024-06-20T21:01:44.013Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 407 | 2024-06-20T21:01:44.013Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 408 | 2024-06-20T21:01:44.013Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 409 | 2024-06-20T21:01:44.017Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 410 | 2024-06-20T21:01:44.050Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 411 | 2024-06-20T21:01:44.058Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_39d4ca15-26b5-4bd0-bbf6-fd95c2268c88/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 412 | 2024-06-20T21:01:44.182Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_39d4ca15-26b5-4bd0-bbf6-fd95c2268c88
file = sled-storage/src/dataset.rs:480
|
| 413 | 2024-06-20T21:01:44.189Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:477
|
| 414 | 2024-06-20T21:01:44.189Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:394
|
| 415 | 2024-06-20T21:01:44.189Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:379
|
| 416 | 2024-06-20T21:01:44.189Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 417 | 2024-06-20T21:01:44.189Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
path = /scratch/u2_8.vdev
|
| 418 | 2024-06-20T21:01:44.199Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev
file = sled-hardware/src/disk.rs:304
|
| 419 | 2024-06-20T21:01:44.199Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = ac756826-7d23-4a48-8f48-471e91447b90 (zpool)
|
| 420 | 2024-06-20T21:01:44.296Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }
|
| 421 | 2024-06-20T21:01:44.304Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/dataset.rs:332
|
| 422 | 2024-06-20T21:01:44.304Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 423 | 2024-06-20T21:01:44.304Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 424 | 2024-06-20T21:01:44.304Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 425 | 2024-06-20T21:01:44.304Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 426 | 2024-06-20T21:01:44.304Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 427 | 2024-06-20T21:01:44.304Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 428 | 2024-06-20T21:01:44.339Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 429 | 2024-06-20T21:01:44.347Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_ac756826-7d23-4a48-8f48-471e91447b90/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 430 | 2024-06-20T21:01:44.474Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_ac756826-7d23-4a48-8f48-471e91447b90
file = sled-storage/src/dataset.rs:480
|
| 431 | 2024-06-20T21:01:44.482Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:477
|
| 432 | 2024-06-20T21:01:44.482Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:394
|
| 433 | 2024-06-20T21:01:44.482Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/.omicron-physical-disks.json.tmp
file = common/src/ledger.rs:199
request = omicron_physical_disks_ensure
|
| 434 | 2024-06-20T21:01:44.482Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 7a99c490-997c-4274-94ca-e12a3ae10436 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 9d01641f-20d1-417f-950c-b1e4c58d23b5 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 435 | 2024-06-20T21:01:44.482Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 436 | 2024-06-20T21:01:44.482Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 437 | 2024-06-20T21:01:44.483Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/.omicron-physical-disks.json.tmp
file = common/src/ledger.rs:199
request = omicron_physical_disks_ensure
|
| 438 | 2024-06-20T21:01:44.483Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 12440499
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:49860
req_id = f79937a2-b6eb-4ec7-acb3-12d8ec697c06
response_code = 200
uri = /omicron-physical-disks
|
| 439 | 2024-06-20T21:01:44.514Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 440 | 2024-06-20T21:01:44.515Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:46282
|
| 441 | 2024-06-20T21:01:44.515Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 442 | 2024-06-20T21:01:44.515Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 443 | 2024-06-20T21:01:44.566Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 444 | 2024-06-20T21:01:44.566Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 445 | 2024-06-20T21:01:44.577Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 8.416199067s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:384
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 446 | 2024-06-20T21:01:44.615Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 447 | 2024-06-20T21:01:44.615Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 448 | 2024-06-20T21:01:44.662Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 449 | 2024-06-20T21:01:44.662Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 450 | 2024-06-20T21:01:44.662Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 451 | 2024-06-20T21:01:44.662Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 452 | 2024-06-20T21:01:44.705Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 453 | 2024-06-20T21:01:44.705Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 454 | 2024-06-20T21:01:44.705Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 455 | 2024-06-20T21:01:44.734Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 456 | 2024-06-20T21:01:44.758Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d
file = illumos-utils/src/zone.rs:315
|
| 457 | 2024-06-20T21:01:44.762Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b
file = illumos-utils/src/zone.rs:315
|
| 458 | 2024-06-20T21:01:44.766Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da
file = illumos-utils/src/zone.rs:315
|
| 459 | 2024-06-20T21:01:44.778Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d
file = illumos-utils/src/zone.rs:354
|
| 460 | 2024-06-20T21:01:44.784Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b
file = illumos-utils/src/zone.rs:354
|
| 461 | 2024-06-20T21:01:44.788Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da
file = illumos-utils/src/zone.rs:354
|
| 462 | 2024-06-20T21:01:45.736Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ebe60f78-650f-482d-ab07-6cce79648e59): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 463 | 2024-06-20T21:01:46.347Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService1"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:3::2"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:3::1"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:3::1]:5353"/>
<propval type="astring" name="dns_address" value="[fd00:1122:3344:3::1]:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 464 | 2024-06-20T21:01:46.347Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }
|
| 465 | 2024-06-20T21:01:46.347Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d
|
| 466 | 2024-06-20T21:01:46.831Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService2"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:1::2"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:1::1"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:1::1]:5353"/>
<propval type="astring" name="dns_address" value="[fd00:1122:3344:1::1]:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 467 | 2024-06-20T21:01:47.311Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }
|
| 468 | 2024-06-20T21:01:47.311Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService3"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:2::2"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:2::1"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:2::1]:5353"/>
<propval type="astring" name="dns_address" value="[fd00:1122:3344:2::1]:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 469 | 2024-06-20T21:01:47.311Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }
|
| 470 | 2024-06-20T21:01:47.319Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d") failed: Property not found. retry in 74.418491ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d
|
| 471 | 2024-06-20T21:01:47.319Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b
|
| 472 | 2024-06-20T21:01:47.323Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da
|
| 473 | 2024-06-20T21:01:47.404Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d") failed: Property not found. retry in 55.371605ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d
|
| 474 | 2024-06-20T21:01:47.448Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b") failed: Property not found. retry in 66.612597ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b
|
| 475 | 2024-06-20T21:01:47.455Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da") failed: Property not found. retry in 50.747725ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da
|
| 476 | 2024-06-20T21:01:47.468Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d") failed: Property not found. retry in 271.969121ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d
|
| 477 | 2024-06-20T21:01:47.526Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da") failed: Property not found. retry in 103.81514ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da
|
| 478 | 2024-06-20T21:01:47.645Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b") failed: Property not found. retry in 141.725778ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b
|
| 479 | 2024-06-20T21:01:47.677Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da") failed: Property not found. retry in 184.423028ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da
|
| 480 | 2024-06-20T21:01:47.770Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d") failed: Property not found. retry in 515.711488ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d
|
| 481 | 2024-06-20T21:01:47.816Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b") failed: Property not found. retry in 134.597131ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b
|
| 482 | 2024-06-20T21:01:47.878Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da") failed: Property not found. retry in 420.705867ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da
|
| 483 | 2024-06-20T21:01:47.962Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b") failed: Property not found. retry in 259.569821ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b
|
| 484 | 2024-06-20T21:01:48.230Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b") failed: Property not found. retry in 576.636171ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b
|
| 485 | 2024-06-20T21:01:48.294Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d") failed: Property not found. retry in 628.682451ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d
|
| 486 | 2024-06-20T21:01:48.306Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da") failed: Property not found. retry in 1.191945989s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da
|
| 487 | 2024-06-20T21:01:48.817Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b") failed: Property not found. retry in 784.494033ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b
|
| 488 | 2024-06-20T21:01:48.934Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d") failed: Property not found. retry in 703.467691ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d
|
| 489 | 2024-06-20T21:01:49.510Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da") failed: Property not found. retry in 1.155306993s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da
|
| 490 | 2024-06-20T21:01:49.610Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b") failed: Property not found. retry in 579.043385ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b
|
| 491 | 2024-06-20T21:01:49.646Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d") failed: Property not found. retry in 1.373379954s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d
|
| 492 | 2024-06-20T21:01:49.735Z | INFO | SledAgent (ServiceManager): Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone
file = sled-agent/src/services.rs:2350
|
| 493 | 2024-06-20T21:01:49.735Z | INFO | SledAgent (ServiceManager): Adding bootstrap address
file = illumos-utils/src/running_zone.rs:626
zone = oxz_switch
|
| 494 | 2024-06-20T21:01:50.197Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b") failed: Property not found. retry in 1.054413869s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b
|
| 495 | 2024-06-20T21:01:50.279Z | INFO | SledAgent (ServiceManager): Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe19:6c8b
file = sled-agent/src/services.rs:2357
|
| 496 | 2024-06-20T21:01:50.285Z | INFO | SledAgent (ServiceManager): Zone not using gateway (even though underlay is up)
file = sled-agent/src/services.rs:2418
|
| 497 | 2024-06-20T21:01:50.405Z | INFO | SledAgent (ServiceManager): Setting up dendrite service
file = sled-agent/src/services.rs:2603
|
| 498 | 2024-06-20T21:01:50.649Z | INFO | SledAgent (ServiceManager): Setting up lldpd service
file = sled-agent/src/services.rs:2767
|
| 499 | 2024-06-20T21:01:50.674Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da") failed: Property not found. retry in 633.285096ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da
|
| 500 | 2024-06-20T21:01:50.835Z | INFO | SledAgent (ServiceManager): Setting up MGS service
file = sled-agent/src/services.rs:2482
|
| 501 | 2024-06-20T21:01:51.029Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d") failed: Property not found. retry in 928.216899ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d
|
| 502 | 2024-06-20T21:01:51.131Z | INFO | SledAgent (ServiceManager): Setting up wicketd service
file = sled-agent/src/services.rs:2517
|
| 503 | 2024-06-20T21:01:51.262Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b") failed: Property not found. retry in 1.37918449s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b
|
| 504 | 2024-06-20T21:01:51.317Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da") failed: Property not found. retry in 590.371413ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da
|
| 505 | 2024-06-20T21:01:51.341Z | INFO | SledAgent (ServiceManager): Setting up mgd service
file = sled-agent/src/services.rs:2817
|
| 506 | 2024-06-20T21:01:51.519Z | INFO | SledAgent (ServiceManager): Setting up mg-ddm service
file = sled-agent/src/services.rs:2844
|
| 507 | 2024-06-20T21:01:51.738Z | INFO | SledAgent (ServiceManager): Setting up tfport service
file = sled-agent/src/services.rs:2711
|
| 508 | 2024-06-20T21:01:51.912Z | INFO | SledAgent (ServiceManager): Setting up Simulated SP service
file = sled-agent/src/services.rs:2511
|
| 509 | 2024-06-20T21:01:51.916Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da") failed: Property not found. retry in 917.862377ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da
|
| 510 | 2024-06-20T21:01:51.921Z | INFO | SledAgent (ServiceManager): Re-enabling running switch zone (new address)
file = sled-agent/src/services.rs:3903
new = [fd00:1122:3344:101::2, ::1]
old = [::1]
|
| 511 | 2024-06-20T21:01:51.921Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists
file = sled-agent/src/services.rs:3919
|
| 512 | 2024-06-20T21:01:51.921Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:609
zone = oxz_switch
|
| 513 | 2024-06-20T21:01:52.479Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists - OK
file = sled-agent/src/services.rs:3927
|
| 514 | 2024-06-20T21:01:52.485Z | INFO | SledAgent (ServiceManager): configuring dendrite service
file = sled-agent/src/services.rs:3983
|
| 515 | 2024-06-20T21:01:52.652Z | INFO | SledAgent (ServiceManager): configuring lldp service
file = sled-agent/src/services.rs:4045
|
| 516 | 2024-06-20T21:01:52.759Z | INFO | SledAgent (ServiceManager): configuring wicketd
file = sled-agent/src/services.rs:4026
rack_subnet = fd00:1122:3344::
|
| 517 | 2024-06-20T21:01:52.781Z | INFO | SledAgent (ServiceManager): configuring mgd service
file = sled-agent/src/services.rs:4073
|
| 518 | 2024-06-20T21:01:52.854Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_internal_dns_b294fa61-9f44-4568-afb9-45e8dfed810d
|
| 519 | 2024-06-20T21:01:52.854Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_internal_dns_8c770d8b-5afa-42fc-ad13-e754ea10ed0b
|
| 520 | 2024-06-20T21:01:52.854Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_internal_dns_a23901b9-ec53-4e80-87b3-8a5db532b3da
|
| 521 | 2024-06-20T21:01:52.854Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 522 | 2024-06-20T21:01:52.854Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 523 | 2024-06-20T21:01:52.855Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 8340042
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46282
req_id = 67424247-15c2-4566-be27-ab6b671a0886
response_code = 204
uri = /omicron-zones
|
| 524 | 2024-06-20T21:01:52.856Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:1::1]:5353
file = sled-agent/src/rack_setup/service.rs:549
|
| 525 | 2024-06-20T21:01:52.881Z | INFO | SledAgent (ServiceManager): configuring mg-ddm service
file = sled-agent/src/services.rs:4100
|
| 526 | 2024-06-20T21:01:53.000Z | INFO | SledAgent (ServiceManager): Determining physical location of our switch zone at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:369
|
| 527 | 2024-06-20T21:01:53.032Z | INFO | SledAgent (ServiceManager): Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:417
|
| 528 | 2024-06-20T21:01:53.045Z | WARN | SledAgent (RSS): failed to write DNS configuration (will retry in 76.688776ms)
dns_config_addr = [fd00:1122:3344:1::1]:5353
error_message = Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:569
|
| 529 | 2024-06-20T21:01:53.064Z | INFO | SledAgent (ServiceManager): Building Port Configuration
file = sled-agent/src/bootstrap/early_networking.rs:656
|
| 530 | 2024-06-20T21:01:53.064Z | INFO | SledAgent (ServiceManager): Checking dendrite uptime
file = sled-agent/src/bootstrap/early_networking.rs:694
|
| 531 | 2024-06-20T21:01:53.065Z | INFO | SledAgent (ServiceManager): Dendrite online
file = sled-agent/src/bootstrap/early_networking.rs:697
uptime = 3
|
| 532 | 2024-06-20T21:01:53.065Z | INFO | SledAgent (ServiceManager): Configuring default uplink on switch
config = PortSettings {\n links: {\n "0": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}
file = sled-agent/src/bootstrap/early_networking.rs:439
|
| 533 | 2024-06-20T21:01:53.144Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:2::1]:5353
file = sled-agent/src/rack_setup/service.rs:549
|
| 534 | 2024-06-20T21:01:53.149Z | INFO | SledAgent (StorageManager): Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }
file = sled-storage/src/manager.rs:381
|
| 535 | 2024-06-20T21:01:53.149Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 536 | 2024-06-20T21:01:53.149Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 537 | 2024-06-20T21:01:53.149Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 538 | 2024-06-20T21:01:53.149Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 539 | 2024-06-20T21:01:53.149Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 540 | 2024-06-20T21:01:53.149Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 541 | 2024-06-20T21:01:53.149Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 542 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 543 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 544 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 545 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 546 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 547 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 548 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 549 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 550 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 551 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 552 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 553 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 554 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 555 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 556 | 2024-06-20T21:01:53.150Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 557 | 2024-06-20T21:01:53.192Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:3::1]:5353
file = sled-agent/src/rack_setup/service.rs:549
|
| 558 | 2024-06-20T21:01:53.239Z | INFO | SledAgent (RSS): Configured all DNS servers
file = sled-agent/src/rack_setup/service.rs:585
|
| 559 | 2024-06-20T21:01:53.239Z | INFO | SledAgent (RSS): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:263
|
| 560 | 2024-06-20T21:01:53.271Z | INFO | SledAgent (RSS): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:280
|
| 561 | 2024-06-20T21:01:53.272Z | INFO | SledAgent (RSS): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:305
|
| 562 | 2024-06-20T21:01:53.272Z | INFO | SledAgent (RSS): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:197
|
| 563 | 2024-06-20T21:01:53.300Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 564 | 2024-06-20T21:01:53.301Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:64154
|
| 565 | 2024-06-20T21:01:53.301Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 566 | 2024-06-20T21:01:53.301Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 567 | 2024-06-20T21:01:53.320Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 568 | 2024-06-20T21:01:53.320Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 569 | 2024-06-20T21:01:53.339Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 570 | 2024-06-20T21:01:53.339Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 571 | 2024-06-20T21:01:53.354Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 572 | 2024-06-20T21:01:53.354Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 573 | 2024-06-20T21:01:53.354Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 574 | 2024-06-20T21:01:53.354Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 575 | 2024-06-20T21:01:53.364Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:263
|
| 576 | 2024-06-20T21:01:53.396Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:280
|
| 577 | 2024-06-20T21:01:53.396Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:305
|
| 578 | 2024-06-20T21:01:53.396Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:197
|
| 579 | 2024-06-20T21:01:53.440Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:393
port = Port { inner: PortInner { name: "opte0", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 192, 81]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: "vopte0" } }
|
| 580 | 2024-06-20T21:01:53.441Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = ntp
|
| 581 | 2024-06-20T21:01:53.441Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 582 | 2024-06-20T21:01:53.441Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 583 | 2024-06-20T21:01:53.441Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 584 | 2024-06-20T21:01:53.443Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 585 | 2024-06-20T21:01:53.468Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12
file = illumos-utils/src/zone.rs:315
|
| 586 | 2024-06-20T21:01:53.486Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12
file = illumos-utils/src/zone.rs:354
|
| 587 | 2024-06-20T21:01:54.104Z | INFO | SledAgent (ServiceManager): Profile for oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService4"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::1b"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/chrony-setup">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="allow" value="fd00:1122:3344:100::/56"/>
<propval type="boolean" name="boundary" value="true"/>
<propval type="astring" name="server" value="0.pool.ntp.org"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="1.1.1.1" />
<value_node value="9.9.9.9" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/ntp">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte0"/>
<propval type="astring" name="gateway" value="172.30.3.1"/>
<propval type="astring" name="ip" value="172.30.3.5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 588 | 2024-06-20T21:01:54.104Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12
|
| 589 | 2024-06-20T21:01:54.231Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12") failed: Property not found. retry in 56.315157ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12
|
| 590 | 2024-06-20T21:01:54.300Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12") failed: Property not found. retry in 108.013911ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12
|
| 591 | 2024-06-20T21:01:54.432Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12") failed: Property not found. retry in 183.818955ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12
|
| 592 | 2024-06-20T21:01:54.626Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12") failed: Property not found. retry in 444.895224ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12
|
| 593 | 2024-06-20T21:01:55.079Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12") failed: Property not found. retry in 853.253587ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12
|
| 594 | 2024-06-20T21:01:55.942Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12") failed: Property not found. retry in 1.207304399s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12
|
| 595 | 2024-06-20T21:01:56.393Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:33416
|
| 596 | 2024-06-20T21:01:56.393Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 92
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:33416
req_id = 982148a9-e04e-4556-affd-37d2254153eb
response_code = 200
uri = /baseboard
|
| 597 | 2024-06-20T21:01:57.158Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12") failed: Property not found. retry in 1.313873826s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12
|
| 598 | 2024-06-20T21:01:57.997Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 36.020634515s
error = "no record found for Query { name: Name(\\"_nexus._tcp.control-plane.oxide.internal.\\"), query_type: SRV, query_class: IN }"
file = oximeter/producer/src/lib.rs:384
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 599 | 2024-06-20T21:01:58.481Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12") failed: Property not found. retry in 518.317771ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12
|
| 600 | 2024-06-20T21:01:59.019Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12
|
| 601 | 2024-06-20T21:01:59.019Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 602 | 2024-06-20T21:01:59.019Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 603 | 2024-06-20T21:01:59.020Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 5718824
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:64154
req_id = 02d2bf21-ba90-4c12-90d1-b295e02e899c
response_code = 204
uri = /omicron-zones
|
| 604 | 2024-06-20T21:01:59.021Z | INFO | SledAgent (RSS): Waiting for rack time synchronization
file = sled-agent/src/rack_setup/service.rs:626
|
| 605 | 2024-06-20T21:01:59.050Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:606
|
| 606 | 2024-06-20T21:01:59.050Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:47934
|
| 607 | 2024-06-20T21:01:59.061Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3633
|
| 608 | 2024-06-20T21:01:59.061Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 11272
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:47934
req_id = f7f944a1-9a3b-4e06-a76f-732dba2fa68c
response_code = 500
uri = /timesync
|
| 609 | 2024-06-20T21:01:59.064Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:656
|
| 610 | 2024-06-20T21:01:59.203Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:606
|
| 611 | 2024-06-20T21:01:59.203Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:51844
|
| 612 | 2024-06-20T21:02:03.213Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3633
|
| 613 | 2024-06-20T21:02:03.213Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 4010246
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:51844
req_id = 13cb390a-7867-41fd-89ae-cd5807bd9bde
response_code = 500
uri = /timesync
|
| 614 | 2024-06-20T21:02:03.216Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:656
|
| 615 | 2024-06-20T21:02:03.406Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:606
|
| 616 | 2024-06-20T21:02:03.406Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:33987
|
| 617 | 2024-06-20T21:02:03.414Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3633
|
| 618 | 2024-06-20T21:02:03.414Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 8385
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:33987
req_id = 91d87dbe-2068-4777-ade2-8fcc51d8f182
response_code = 500
uri = /timesync
|
| 619 | 2024-06-20T21:02:03.417Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:656
|
| 620 | 2024-06-20T21:02:03.536Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:606
|
| 621 | 2024-06-20T21:02:03.536Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:61039
|
| 622 | 2024-06-20T21:02:07.545Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_9c3e0325-17c9-41b6-9df0-8049b765cb12': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3633
|
| 623 | 2024-06-20T21:02:07.545Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 4009215
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:61039
req_id = 59154800-0109-402f-831a-b91557f3d0b3
response_code = 500
uri = /timesync
|
| 624 | 2024-06-20T21:02:07.547Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:656
|
| 625 | 2024-06-20T21:02:07.784Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:606
|
| 626 | 2024-06-20T21:02:07.784Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:60662
|
| 627 | 2024-06-20T21:02:07.792Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 8284
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:60662
req_id = ed516a95-27e6-4419-af68-9c4855428b29
response_code = 200
uri = /timesync
|
| 628 | 2024-06-20T21:02:07.794Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:634
|
| 629 | 2024-06-20T21:02:07.794Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:656
|
| 630 | 2024-06-20T21:02:07.990Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:606
|
| 631 | 2024-06-20T21:02:07.990Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:36973
|
| 632 | 2024-06-20T21:02:07.999Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 8249
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:36973
req_id = 8d5fb750-dcd3-4964-82ad-72378ff0ce87
response_code = 200
uri = /timesync
|
| 633 | 2024-06-20T21:02:08.000Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:634
|
| 634 | 2024-06-20T21:02:08.000Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:656
|
| 635 | 2024-06-20T21:02:08.367Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:606
|
| 636 | 2024-06-20T21:02:08.367Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:38701
|
| 637 | 2024-06-20T21:02:08.375Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 7812
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:38701
req_id = 4392603b-36d6-4be2-b4a6-8eca474ecedf
response_code = 200
uri = /timesync
|
| 638 | 2024-06-20T21:02:08.376Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:634
|
| 639 | 2024-06-20T21:02:08.376Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:656
|
| 640 | 2024-06-20T21:02:08.612Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:606
|
| 641 | 2024-06-20T21:02:08.612Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:42880
|
| 642 | 2024-06-20T21:02:08.620Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 7659
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:42880
req_id = d6b63ff7-1ac3-4e3f-89e1-acabcdf56931
response_code = 200
uri = /timesync
|
| 643 | 2024-06-20T21:02:08.621Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:634
|
| 644 | 2024-06-20T21:02:08.621Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:656
|
| 645 | 2024-06-20T21:02:09.026Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:606
|
| 646 | 2024-06-20T21:02:09.026Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:63761
|
| 647 | 2024-06-20T21:02:09.034Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 7747
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:63761
req_id = 7fcddcd4-6cea-421b-8e25-8676142e47a2
response_code = 200
uri = /timesync
|
| 648 | 2024-06-20T21:02:09.035Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:634
|
| 649 | 2024-06-20T21:02:09.035Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:656
|
| 650 | 2024-06-20T21:02:09.329Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:606
|
| 651 | 2024-06-20T21:02:09.329Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:33481
|
| 652 | 2024-06-20T21:02:09.337Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 8019
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:33481
req_id = df7ac792-8d03-475e-8e24-281eb91baa10
response_code = 200
uri = /timesync
|
| 653 | 2024-06-20T21:02:09.338Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:634
|
| 654 | 2024-06-20T21:02:09.338Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:656
|
| 655 | 2024-06-20T21:02:09.664Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:606
|
| 656 | 2024-06-20T21:02:09.664Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:55945
|
| 657 | 2024-06-20T21:02:09.673Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 8303
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:55945
req_id = a8266f24-5dd1-45b8-a5a6-3ab5449060ee
response_code = 200
uri = /timesync
|
| 658 | 2024-06-20T21:02:09.675Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:634
|
| 659 | 2024-06-20T21:02:09.675Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:656
|
| 660 | 2024-06-20T21:02:09.614Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:606
|
| 661 | 2024-06-20T21:02:09.614Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:55904
|
| 662 | 2024-06-20T21:02:09.634Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 19258
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:55904
req_id = b1fd3a95-1be2-48be-ba8e-81085d137188
response_code = 200
uri = /timesync
|
| 663 | 2024-06-20T21:02:09.637Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 3638230730, ip_addr: 216.218.254.202, stratum: 3, ref_time: 1718917329.1247766, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:634
|
| 664 | 2024-06-20T21:02:09.637Z | INFO | SledAgent (RSS): Finished setting up Internal DNS and NTP
file = sled-agent/src/rack_setup/service.rs:1224
|
| 665 | 2024-06-20T21:02:09.666Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 666 | 2024-06-20T21:02:09.666Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:33071
|
| 667 | 2024-06-20T21:02:09.666Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 668 | 2024-06-20T21:02:09.666Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 669 | 2024-06-20T21:02:09.686Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 670 | 2024-06-20T21:02:09.686Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 671 | 2024-06-20T21:02:09.705Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 672 | 2024-06-20T21:02:09.705Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 673 | 2024-06-20T21:02:09.723Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 674 | 2024-06-20T21:02:09.723Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 675 | 2024-06-20T21:02:09.774Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 676 | 2024-06-20T21:02:09.774Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 677 | 2024-06-20T21:02:09.825Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 678 | 2024-06-20T21:02:09.825Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 679 | 2024-06-20T21:02:09.876Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 680 | 2024-06-20T21:02:09.876Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 681 | 2024-06-20T21:02:09.926Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 682 | 2024-06-20T21:02:09.927Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 683 | 2024-06-20T21:02:09.980Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 684 | 2024-06-20T21:02:09.980Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 685 | 2024-06-20T21:02:09.980Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 686 | 2024-06-20T21:02:09.980Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 687 | 2024-06-20T21:02:10.064Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 688 | 2024-06-20T21:02:10.064Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 689 | 2024-06-20T21:02:10.064Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 690 | 2024-06-20T21:02:10.064Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 691 | 2024-06-20T21:02:10.064Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 692 | 2024-06-20T21:02:10.150Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
file = illumos-utils/src/zone.rs:315
|
| 693 | 2024-06-20T21:02:10.154Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
file = illumos-utils/src/zone.rs:315
|
| 694 | 2024-06-20T21:02:10.158Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
file = illumos-utils/src/zone.rs:315
|
| 695 | 2024-06-20T21:02:10.162Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
file = illumos-utils/src/zone.rs:315
|
| 696 | 2024-06-20T21:02:10.166Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
file = illumos-utils/src/zone.rs:315
|
| 697 | 2024-06-20T21:02:10.172Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
file = illumos-utils/src/zone.rs:354
|
| 698 | 2024-06-20T21:02:10.177Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
file = illumos-utils/src/zone.rs:354
|
| 699 | 2024-06-20T21:02:10.182Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
file = illumos-utils/src/zone.rs:354
|
| 700 | 2024-06-20T21:02:10.186Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
file = illumos-utils/src/zone.rs:354
|
| 701 | 2024-06-20T21:02:10.190Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
file = illumos-utils/src/zone.rs:354
|
| 702 | 2024-06-20T21:02:13.764Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService5"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::6"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="[fd00:1122:3344:101::6]:32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value="f71b4047-e660-4f04-b1b8-cae0ee49769f"/>
<propval type="astring" name="cockroach_address" value="[fd00:1122:3344:101::6]:32221"/>
<propval type="astring" name="http_address" value="[fd00:1122:3344:101::6]:32222"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 703 | 2024-06-20T21:02:13.765Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
|
| 704 | 2024-06-20T21:02:13.773Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService7"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::4"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="[fd00:1122:3344:101::4]:32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value="021a4838-0953-41f3-a0c1-d3ce533ca2ae"/>
<propval type="astring" name="cockroach_address" value="[fd00:1122:3344:101::4]:32221"/>
<propval type="astring" name="http_address" value="[fd00:1122:3344:101::4]:32222"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 705 | 2024-06-20T21:02:13.773Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 706 | 2024-06-20T21:02:13.874Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService9"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::7"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="[fd00:1122:3344:101::7]:32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value="b9e1c26a-87d3-4501-a588-62f9673bcb7b"/>
<propval type="astring" name="cockroach_address" value="[fd00:1122:3344:101::7]:32221"/>
<propval type="astring" name="http_address" value="[fd00:1122:3344:101::7]:32222"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 707 | 2024-06-20T21:02:13.874Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 708 | 2024-06-20T21:02:13.913Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f") failed: Property not found. retry in 59.763298ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
|
| 709 | 2024-06-20T21:02:13.920Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 67.412729ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 710 | 2024-06-20T21:02:13.934Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService8"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::3"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="[fd00:1122:3344:101::3]:32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value="541f1717-17b0-4191-be31-eab4c1500570"/>
<propval type="astring" name="cockroach_address" value="[fd00:1122:3344:101::3]:32221"/>
<propval type="astring" name="http_address" value="[fd00:1122:3344:101::3]:32222"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 711 | 2024-06-20T21:02:13.934Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
|
| 712 | 2024-06-20T21:02:13.941Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService6"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="[fd00:1122:3344:101::5]:32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value="450b3b5f-7293-4a8e-978d-a22e72953896"/>
<propval type="astring" name="cockroach_address" value="[fd00:1122:3344:101::5]:32221"/>
<propval type="astring" name="http_address" value="[fd00:1122:3344:101::5]:32222"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 713 | 2024-06-20T21:02:13.941Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 714 | 2024-06-20T21:02:13.992Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f") failed: Property not found. retry in 144.521231ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
|
| 715 | 2024-06-20T21:02:14.105Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 78.135445ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 716 | 2024-06-20T21:02:14.208Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f") failed: Property not found. retry in 286.223664ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
|
| 717 | 2024-06-20T21:02:14.271Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 177.939059ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 718 | 2024-06-20T21:02:14.542Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 36.742279ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 719 | 2024-06-20T21:02:14.619Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f") failed: Property not found. retry in 270.710589ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
|
| 720 | 2024-06-20T21:02:14.693Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 51.925336ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 721 | 2024-06-20T21:02:14.801Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 406.283528ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 722 | 2024-06-20T21:02:14.902Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570") failed: Property not found. retry in 33.276679ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
|
| 723 | 2024-06-20T21:02:15.036Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 69.718523ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 724 | 2024-06-20T21:02:15.138Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f") failed: Property not found. retry in 939.473762ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
|
| 725 | 2024-06-20T21:02:15.257Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 58.364608ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 726 | 2024-06-20T21:02:15.268Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570") failed: Property not found. retry in 100.383646ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
|
| 727 | 2024-06-20T21:02:15.752Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 108.869202ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 728 | 2024-06-20T21:02:15.909Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 261.593953ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 729 | 2024-06-20T21:02:15.965Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 996.40411ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 730 | 2024-06-20T21:02:16.117Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570") failed: Property not found. retry in 167.728465ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
|
| 731 | 2024-06-20T21:02:16.167Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 412.852317ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 732 | 2024-06-20T21:02:16.211Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f") failed: Property not found. retry in 577.767747ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
|
| 733 | 2024-06-20T21:02:16.229Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 467.111381ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 734 | 2024-06-20T21:02:16.298Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570") failed: Property not found. retry in 283.894056ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
|
| 735 | 2024-06-20T21:02:16.589Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 853.617669ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 736 | 2024-06-20T21:02:16.598Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570") failed: Property not found. retry in 828.044917ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
|
| 737 | 2024-06-20T21:02:16.708Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 1.024037082s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 738 | 2024-06-20T21:02:16.802Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f") failed: Property not found. retry in 1.416883174s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
|
| 739 | 2024-06-20T21:02:16.974Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 1.155038169s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 740 | 2024-06-20T21:02:17.435Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570") failed: Property not found. retry in 919.979814ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
|
| 741 | 2024-06-20T21:02:17.453Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 1.499187823s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 742 | 2024-06-20T21:02:17.741Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 753.972165ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 743 | 2024-06-20T21:02:18.138Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 725.28543ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 744 | 2024-06-20T21:02:18.227Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f") failed: Property not found. retry in 1.100052755s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
|
| 745 | 2024-06-20T21:02:18.368Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570") failed: Property not found. retry in 973.231709ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
|
| 746 | 2024-06-20T21:02:18.507Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 690.581961ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 747 | 2024-06-20T21:02:18.874Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 1.492678584s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 748 | 2024-06-20T21:02:18.961Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 1.203192148s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 749 | 2024-06-20T21:02:19.210Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 673.59618ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 750 | 2024-06-20T21:02:19.336Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f") failed: Property not found. retry in 758.144052ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
|
| 751 | 2024-06-20T21:02:19.349Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570") failed: Property not found. retry in 539.906432ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
|
| 752 | 2024-06-20T21:02:19.892Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 1.31439668s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 753 | 2024-06-20T21:02:19.899Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570") failed: Property not found. retry in 821.919489ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
|
| 754 | 2024-06-20T21:02:20.103Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f") failed: Property not found. retry in 1.483304427s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
|
| 755 | 2024-06-20T21:02:20.173Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 877.519125ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 756 | 2024-06-20T21:02:20.375Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 1.140103438s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 757 | 2024-06-20T21:02:20.731Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570") failed: Property not found. retry in 669.351291ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
|
| 758 | 2024-06-20T21:02:21.060Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 1.490065032s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 759 | 2024-06-20T21:02:21.216Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 1.13747416s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 760 | 2024-06-20T21:02:21.410Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570") failed: Property not found. retry in 1.303573211s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
|
| 761 | 2024-06-20T21:02:21.524Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 954.901119ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 762 | 2024-06-20T21:02:21.595Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f") failed: Property not found. retry in 938.051825ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
|
| 763 | 2024-06-20T21:02:22.363Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 898.111744ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 764 | 2024-06-20T21:02:22.486Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 1.05072903s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 765 | 2024-06-20T21:02:22.558Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 1.164856721s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 766 | 2024-06-20T21:02:23.271Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 1.191466665s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 767 | 2024-06-20T21:02:23.547Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 798.375811ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 768 | 2024-06-20T21:02:23.731Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 944.110447ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 769 | 2024-06-20T21:02:24.355Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 789.899291ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 770 | 2024-06-20T21:02:24.470Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 512.70545ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 771 | 2024-06-20T21:02:24.684Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 540.860866ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 772 | 2024-06-20T21:02:24.990Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 765.5048ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 773 | 2024-06-20T21:02:25.152Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 1.284116367s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 774 | 2024-06-20T21:02:25.232Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 975.292016ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 775 | 2024-06-20T21:02:25.765Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 923.626154ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 776 | 2024-06-20T21:02:26.216Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 788.972362ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 777 | 2024-06-20T21:02:26.326Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:49724
|
| 778 | 2024-06-20T21:02:26.326Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 66
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:49724
req_id = 37af4df5-91bc-4d5d-b684-0383d8444bcb
response_code = 200
uri = /baseboard
|
| 779 | 2024-06-20T21:02:26.444Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 679.277825ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 780 | 2024-06-20T21:02:26.697Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 931.190516ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 781 | 2024-06-20T21:02:27.013Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 1.263702753s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 782 | 2024-06-20T21:02:27.133Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 554.278778ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 783 | 2024-06-20T21:02:27.637Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 598.516565ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 784 | 2024-06-20T21:02:27.696Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 1.397183394s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 785 | 2024-06-20T21:02:28.245Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 1.038094625s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 786 | 2024-06-20T21:02:28.285Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 1.460609856s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 787 | 2024-06-20T21:02:29.103Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 560.590193ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 788 | 2024-06-20T21:02:29.291Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 867.804965ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 789 | 2024-06-20T21:02:29.672Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 518.972682ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 790 | 2024-06-20T21:02:29.756Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 1.160224075s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 791 | 2024-06-20T21:02:30.168Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 914.353042ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 792 | 2024-06-20T21:02:30.198Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 853.387179ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 793 | 2024-06-20T21:02:30.925Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 936.653127ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 794 | 2024-06-20T21:02:31.061Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae") failed: Property not found. retry in 1.065822443s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 795 | 2024-06-20T21:02:31.092Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896") failed: Property not found. retry in 1.084750161s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 796 | 2024-06-20T21:02:31.871Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b") failed: Property not found. retry in 918.875973ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 797 | 2024-06-20T21:02:32.812Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_cockroachdb_f71b4047-e660-4f04-b1b8-cae0ee49769f
|
| 798 | 2024-06-20T21:02:32.812Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_cockroachdb_450b3b5f-7293-4a8e-978d-a22e72953896
|
| 799 | 2024-06-20T21:02:32.812Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae
|
| 800 | 2024-06-20T21:02:32.812Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_cockroachdb_541f1717-17b0-4191-be31-eab4c1500570
|
| 801 | 2024-06-20T21:02:32.812Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_cockroachdb_b9e1c26a-87d3-4501-a588-62f9673bcb7b
|
| 802 | 2024-06-20T21:02:32.812Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 803 | 2024-06-20T21:02:32.812Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 804 | 2024-06-20T21:02:32.813Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 22487285
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33071
req_id = 843edccf-0457-4950-9702-50349486c2f9
response_code = 204
uri = /omicron-zones
|
| 805 | 2024-06-20T21:02:32.844Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:34300
|
| 806 | 2024-06-20T21:02:32.870Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 26590
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:34300
req_id = 23b1a9cb-63d3-48de-b739-c072726a08ae
response_code = 500
uri = /cockroachdb
|
| 807 | 2024-06-20T21:02:32.871Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "23b1a9cb-63d3-48de-b739-c072726a08ae", "content-length": "124", "date": "Thu, 20 Jun 2024 21:02:32 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "23b1a9cb-63d3-48de-b739-c072726a08ae" }
file = sled-agent/src/rack_setup/service.rs:966
retry_after = 61.233475ms
|
| 808 | 2024-06-20T21:02:32.964Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_021a4838-0953-41f3-a0c1-d3ce533ca2ae /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 29852
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:34300
req_id = d2d48075-627d-4284-a375-309e0f2819fd
response_code = 500
uri = /cockroachdb
|
| 809 | 2024-06-20T21:02:32.964Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d2d48075-627d-4284-a375-309e0f2819fd", "content-length": "124", "date": "Thu, 20 Jun 2024 21:02:32 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d2d48075-627d-4284-a375-309e0f2819fd" }
file = sled-agent/src/rack_setup/service.rs:966
retry_after = 174.149758ms
|
| 810 | 2024-06-20T21:02:33.169Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::4]:32221
file = sled-agent/src/services.rs:3467
|
| 811 | 2024-06-20T21:02:48.565Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3485
|
| 812 | 2024-06-20T21:03:02.659Z | INFO | SledAgent (ServiceManager): Formatting CRDB - Completed
file = sled-agent/src/services.rs:3508
|
| 813 | 2024-06-20T21:03:02.660Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 29520470
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:34300
req_id = d88b2f72-03c5-441e-af36-90f7fd917438
response_code = 204
uri = /cockroachdb
|
| 814 | 2024-06-20T21:03:02.661Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:32928
|
| 815 | 2024-06-20T21:03:02.662Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 663
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:32928
req_id = 957774ae-73ed-4502-b923-b820aaa51418
response_code = 200
uri = /baseboard
|
| 816 | 2024-06-20T21:03:02.730Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4e960e23-20ba-48e5-9697-e299814c1217, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ef10530a-0825-4479-8014-6a7d3078262d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 25b89476-359e-47f8-bea2-3de9e5e99194, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 271e109e-182e-4626-b37a-fed0542ab9a5, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 70ea324e-1d3b-451f-92e2-3dd4893ba7df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: fe5b0415-63cc-4561-8e18-8b38bd5aa235, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 646a904f-1208-4787-8b83-6ad524b58ce3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } } } }, OmicronZoneConfig { id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } } } }, OmicronZoneConfig { id: bc18ef60-2412-4f2c-9cab-691f92a94385, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 817 | 2024-06-20T21:03:02.731Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:63067
|
| 818 | 2024-06-20T21:03:02.732Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 819 | 2024-06-20T21:03:02.732Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 820 | 2024-06-20T21:03:02.755Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 821 | 2024-06-20T21:03:02.755Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 822 | 2024-06-20T21:03:02.778Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 823 | 2024-06-20T21:03:02.778Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 824 | 2024-06-20T21:03:02.797Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 825 | 2024-06-20T21:03:02.797Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 826 | 2024-06-20T21:03:02.816Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 827 | 2024-06-20T21:03:02.816Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 828 | 2024-06-20T21:03:02.834Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 829 | 2024-06-20T21:03:02.834Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 830 | 2024-06-20T21:03:02.852Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 831 | 2024-06-20T21:03:02.852Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 832 | 2024-06-20T21:03:02.871Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 833 | 2024-06-20T21:03:02.871Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 834 | 2024-06-20T21:03:02.891Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 835 | 2024-06-20T21:03:02.892Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 836 | 2024-06-20T21:03:03.038Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 837 | 2024-06-20T21:03:03.038Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 838 | 2024-06-20T21:03:03.155Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 839 | 2024-06-20T21:03:03.155Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 840 | 2024-06-20T21:03:03.256Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 841 | 2024-06-20T21:03:03.256Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 842 | 2024-06-20T21:03:03.347Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 843 | 2024-06-20T21:03:03.347Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 844 | 2024-06-20T21:03:03.456Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 845 | 2024-06-20T21:03:03.456Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 846 | 2024-06-20T21:03:03.548Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 847 | 2024-06-20T21:03:03.548Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 848 | 2024-06-20T21:03:03.658Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 849 | 2024-06-20T21:03:03.659Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 850 | 2024-06-20T21:03:03.747Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 851 | 2024-06-20T21:03:03.747Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 852 | 2024-06-20T21:03:03.838Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 853 | 2024-06-20T21:03:03.838Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 854 | 2024-06-20T21:03:03.932Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 855 | 2024-06-20T21:03:03.932Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 856 | 2024-06-20T21:03:04.029Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 857 | 2024-06-20T21:03:04.029Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 858 | 2024-06-20T21:03:04.118Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 859 | 2024-06-20T21:03:04.118Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 860 | 2024-06-20T21:03:04.118Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 861 | 2024-06-20T21:03:04.118Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 862 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 863 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 864 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 865 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 866 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 867 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 868 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 869 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 870 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 871 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 872 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 873 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 874 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 875 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 876 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 877 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 878 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 879 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 880 | 2024-06-20T21:03:04.236Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 881 | 2024-06-20T21:03:04.236Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d", TimeNotSynchronized), ("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec", TimeNotSynchronized), ("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194", TimeNotSynchronized), ("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235", TimeNotSynchronized), ("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4", TimeNotSynchronized), ("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385", TimeNotSynchronized), ("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df", TimeNotSynchronized), ("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc", TimeNotSynchronized), ("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3", TimeNotSynchronized), ("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958", TimeNotSynchronized), ("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9", TimeNotSynchronized), ("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5", TimeNotSynchronized), ("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0", TimeNotSynchronized), ("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217", TimeNotSynchronized), ("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9", TimeNotSynchronized), ("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57", TimeNotSynchronized), ("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc", TimeNotSynchronized), ("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f", TimeNotSynchronized), ("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 1504667
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:63067
req_id = 330a8922-b9dd-4829-bfc6-f2c18cf747ee
response_code = 500
uri = /omicron-zones
|
| 882 | 2024-06-20T21:03:04.240Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "330a8922-b9dd-4829-bfc6-f2c18cf747ee", "content-length": "124", "date": "Thu, 20 Jun 2024 21:03:04 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "330a8922-b9dd-4829-bfc6-f2c18cf747ee" }
file = sled-agent/src/rack_setup/service.rs:469
retry_after = 51.809413ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 883 | 2024-06-20T21:03:04.293Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4e960e23-20ba-48e5-9697-e299814c1217, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ef10530a-0825-4479-8014-6a7d3078262d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 25b89476-359e-47f8-bea2-3de9e5e99194, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 271e109e-182e-4626-b37a-fed0542ab9a5, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 70ea324e-1d3b-451f-92e2-3dd4893ba7df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: fe5b0415-63cc-4561-8e18-8b38bd5aa235, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 646a904f-1208-4787-8b83-6ad524b58ce3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } } } }, OmicronZoneConfig { id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } } } }, OmicronZoneConfig { id: bc18ef60-2412-4f2c-9cab-691f92a94385, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 884 | 2024-06-20T21:03:04.293Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 885 | 2024-06-20T21:03:04.293Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 886 | 2024-06-20T21:03:04.317Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 887 | 2024-06-20T21:03:04.317Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 888 | 2024-06-20T21:03:04.339Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 889 | 2024-06-20T21:03:04.339Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 890 | 2024-06-20T21:03:04.358Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 891 | 2024-06-20T21:03:04.358Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 892 | 2024-06-20T21:03:04.377Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 893 | 2024-06-20T21:03:04.377Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 894 | 2024-06-20T21:03:04.397Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 895 | 2024-06-20T21:03:04.397Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 896 | 2024-06-20T21:03:04.417Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 897 | 2024-06-20T21:03:04.417Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 898 | 2024-06-20T21:03:04.437Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 899 | 2024-06-20T21:03:04.437Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 900 | 2024-06-20T21:03:04.460Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 901 | 2024-06-20T21:03:04.460Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 902 | 2024-06-20T21:03:04.480Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 903 | 2024-06-20T21:03:04.480Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 904 | 2024-06-20T21:03:04.501Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 905 | 2024-06-20T21:03:04.501Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 906 | 2024-06-20T21:03:04.521Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 907 | 2024-06-20T21:03:04.521Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 908 | 2024-06-20T21:03:04.541Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 909 | 2024-06-20T21:03:04.541Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 910 | 2024-06-20T21:03:04.561Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 911 | 2024-06-20T21:03:04.561Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 912 | 2024-06-20T21:03:04.580Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 913 | 2024-06-20T21:03:04.580Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 914 | 2024-06-20T21:03:04.601Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 915 | 2024-06-20T21:03:04.601Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 916 | 2024-06-20T21:03:04.620Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 917 | 2024-06-20T21:03:04.620Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 918 | 2024-06-20T21:03:04.642Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 919 | 2024-06-20T21:03:04.642Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 920 | 2024-06-20T21:03:04.663Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 921 | 2024-06-20T21:03:04.663Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 922 | 2024-06-20T21:03:04.684Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 923 | 2024-06-20T21:03:04.684Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 924 | 2024-06-20T21:03:04.704Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 925 | 2024-06-20T21:03:04.704Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 926 | 2024-06-20T21:03:04.705Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 927 | 2024-06-20T21:03:04.705Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 928 | 2024-06-20T21:03:04.803Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 929 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 930 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 931 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 932 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 933 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 934 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 935 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 936 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 937 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 938 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 939 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 940 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 941 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 942 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 943 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 944 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 945 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 946 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 947 | 2024-06-20T21:03:04.804Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57", TimeNotSynchronized), ("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9", TimeNotSynchronized), ("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4", TimeNotSynchronized), ("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec", TimeNotSynchronized), ("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235", TimeNotSynchronized), ("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3", TimeNotSynchronized), ("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194", TimeNotSynchronized), ("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d", TimeNotSynchronized), ("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5", TimeNotSynchronized), ("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385", TimeNotSynchronized), ("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0", TimeNotSynchronized), ("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df", TimeNotSynchronized), ("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4", TimeNotSynchronized), ("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f", TimeNotSynchronized), ("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958", TimeNotSynchronized), ("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9", TimeNotSynchronized), ("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc", TimeNotSynchronized), ("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc", TimeNotSynchronized), ("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 510664
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:63067
req_id = 13b69772-5fb3-4c94-96d3-d901982b7cf8
response_code = 500
uri = /omicron-zones
|
| 948 | 2024-06-20T21:03:04.804Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "13b69772-5fb3-4c94-96d3-d901982b7cf8", "content-length": "124", "date": "Thu, 20 Jun 2024 21:03:04 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "13b69772-5fb3-4c94-96d3-d901982b7cf8" }
file = sled-agent/src/rack_setup/service.rs:469
retry_after = 156.974597ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 949 | 2024-06-20T21:03:04.962Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4e960e23-20ba-48e5-9697-e299814c1217, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ef10530a-0825-4479-8014-6a7d3078262d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 25b89476-359e-47f8-bea2-3de9e5e99194, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 271e109e-182e-4626-b37a-fed0542ab9a5, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 70ea324e-1d3b-451f-92e2-3dd4893ba7df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: fe5b0415-63cc-4561-8e18-8b38bd5aa235, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 646a904f-1208-4787-8b83-6ad524b58ce3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } } } }, OmicronZoneConfig { id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } } } }, OmicronZoneConfig { id: bc18ef60-2412-4f2c-9cab-691f92a94385, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 950 | 2024-06-20T21:03:04.963Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 951 | 2024-06-20T21:03:04.963Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 952 | 2024-06-20T21:03:04.986Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 953 | 2024-06-20T21:03:04.986Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 954 | 2024-06-20T21:03:05.005Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 955 | 2024-06-20T21:03:05.005Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 956 | 2024-06-20T21:03:05.024Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 957 | 2024-06-20T21:03:05.024Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 958 | 2024-06-20T21:03:05.041Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 959 | 2024-06-20T21:03:05.041Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 960 | 2024-06-20T21:03:05.058Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 961 | 2024-06-20T21:03:05.058Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 962 | 2024-06-20T21:03:05.075Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 963 | 2024-06-20T21:03:05.075Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 964 | 2024-06-20T21:03:05.093Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 965 | 2024-06-20T21:03:05.093Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 966 | 2024-06-20T21:03:05.109Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 967 | 2024-06-20T21:03:05.109Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 968 | 2024-06-20T21:03:05.126Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 969 | 2024-06-20T21:03:05.126Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 970 | 2024-06-20T21:03:05.143Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 971 | 2024-06-20T21:03:05.143Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 972 | 2024-06-20T21:03:05.161Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 973 | 2024-06-20T21:03:05.161Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 974 | 2024-06-20T21:03:05.178Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 975 | 2024-06-20T21:03:05.178Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 976 | 2024-06-20T21:03:05.195Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 977 | 2024-06-20T21:03:05.195Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 978 | 2024-06-20T21:03:05.213Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 979 | 2024-06-20T21:03:05.213Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 980 | 2024-06-20T21:03:05.234Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 981 | 2024-06-20T21:03:05.234Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 982 | 2024-06-20T21:03:05.254Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 983 | 2024-06-20T21:03:05.254Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 984 | 2024-06-20T21:03:05.273Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 985 | 2024-06-20T21:03:05.273Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 986 | 2024-06-20T21:03:05.289Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 987 | 2024-06-20T21:03:05.289Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 988 | 2024-06-20T21:03:05.306Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 989 | 2024-06-20T21:03:05.306Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 990 | 2024-06-20T21:03:05.324Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 991 | 2024-06-20T21:03:05.324Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 992 | 2024-06-20T21:03:05.324Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 993 | 2024-06-20T21:03:05.324Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 994 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 995 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 996 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 997 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 998 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 999 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1000 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1001 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1002 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1003 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 1004 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1005 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1006 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1007 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 1008 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 1009 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 1010 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1011 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1012 | 2024-06-20T21:03:05.419Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1013 | 2024-06-20T21:03:05.419Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217", TimeNotSynchronized), ("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9", TimeNotSynchronized), ("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57", TimeNotSynchronized), ("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9", TimeNotSynchronized), ("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d", TimeNotSynchronized), ("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5", TimeNotSynchronized), ("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4", TimeNotSynchronized), ("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec", TimeNotSynchronized), ("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3", TimeNotSynchronized), ("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df", TimeNotSynchronized), ("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc", TimeNotSynchronized), ("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0", TimeNotSynchronized), ("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385", TimeNotSynchronized), ("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958", TimeNotSynchronized), ("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194", TimeNotSynchronized), ("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f", TimeNotSynchronized), ("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc", TimeNotSynchronized), ("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4", TimeNotSynchronized), ("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 456873
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:63067
req_id = d4485782-a563-4a6c-a008-75ba18bf30f9
response_code = 500
uri = /omicron-zones
|
| 1014 | 2024-06-20T21:03:05.420Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d4485782-a563-4a6c-a008-75ba18bf30f9", "content-length": "124", "date": "Thu, 20 Jun 2024 21:03:05 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d4485782-a563-4a6c-a008-75ba18bf30f9" }
file = sled-agent/src/rack_setup/service.rs:469
retry_after = 105.244603ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1015 | 2024-06-20T21:03:05.527Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4e960e23-20ba-48e5-9697-e299814c1217, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ef10530a-0825-4479-8014-6a7d3078262d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 25b89476-359e-47f8-bea2-3de9e5e99194, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 271e109e-182e-4626-b37a-fed0542ab9a5, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 70ea324e-1d3b-451f-92e2-3dd4893ba7df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: fe5b0415-63cc-4561-8e18-8b38bd5aa235, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 646a904f-1208-4787-8b83-6ad524b58ce3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } } } }, OmicronZoneConfig { id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } } } }, OmicronZoneConfig { id: bc18ef60-2412-4f2c-9cab-691f92a94385, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1016 | 2024-06-20T21:03:05.527Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1017 | 2024-06-20T21:03:05.527Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1018 | 2024-06-20T21:03:05.550Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1019 | 2024-06-20T21:03:05.550Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1020 | 2024-06-20T21:03:05.570Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1021 | 2024-06-20T21:03:05.570Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1022 | 2024-06-20T21:03:05.589Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1023 | 2024-06-20T21:03:05.589Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1024 | 2024-06-20T21:03:05.608Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1025 | 2024-06-20T21:03:05.608Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1026 | 2024-06-20T21:03:05.626Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1027 | 2024-06-20T21:03:05.626Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1028 | 2024-06-20T21:03:05.644Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1029 | 2024-06-20T21:03:05.644Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1030 | 2024-06-20T21:03:05.663Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1031 | 2024-06-20T21:03:05.663Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1032 | 2024-06-20T21:03:05.682Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1033 | 2024-06-20T21:03:05.682Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1034 | 2024-06-20T21:03:05.701Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1035 | 2024-06-20T21:03:05.701Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1036 | 2024-06-20T21:03:05.723Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1037 | 2024-06-20T21:03:05.723Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1038 | 2024-06-20T21:03:05.743Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1039 | 2024-06-20T21:03:05.743Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1040 | 2024-06-20T21:03:05.762Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1041 | 2024-06-20T21:03:05.762Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1042 | 2024-06-20T21:03:05.780Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1043 | 2024-06-20T21:03:05.780Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1044 | 2024-06-20T21:03:05.799Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1045 | 2024-06-20T21:03:05.799Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1046 | 2024-06-20T21:03:05.821Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1047 | 2024-06-20T21:03:05.821Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1048 | 2024-06-20T21:03:05.839Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1049 | 2024-06-20T21:03:05.839Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1050 | 2024-06-20T21:03:05.860Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1051 | 2024-06-20T21:03:05.860Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1052 | 2024-06-20T21:03:05.879Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1053 | 2024-06-20T21:03:05.879Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1054 | 2024-06-20T21:03:05.897Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1055 | 2024-06-20T21:03:05.897Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1056 | 2024-06-20T21:03:05.919Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1057 | 2024-06-20T21:03:05.919Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1058 | 2024-06-20T21:03:05.919Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1059 | 2024-06-20T21:03:05.919Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1060 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1061 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1062 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 1063 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1064 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 1065 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1066 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 1067 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1068 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1069 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1070 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1071 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1072 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 1073 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 1074 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 1075 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1076 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1077 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1078 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 1079 | 2024-06-20T21:03:06.012Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235", TimeNotSynchronized), ("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4", TimeNotSynchronized), ("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9", TimeNotSynchronized), ("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc", TimeNotSynchronized), ("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f", TimeNotSynchronized), ("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec", TimeNotSynchronized), ("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194", TimeNotSynchronized), ("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3", TimeNotSynchronized), ("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0", TimeNotSynchronized), ("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4", TimeNotSynchronized), ("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d", TimeNotSynchronized), ("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9", TimeNotSynchronized), ("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57", TimeNotSynchronized), ("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958", TimeNotSynchronized), ("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df", TimeNotSynchronized), ("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385", TimeNotSynchronized), ("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5", TimeNotSynchronized), ("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc", TimeNotSynchronized), ("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 485072
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:63067
req_id = 272a606c-16df-421b-9f4b-b84df51981f3
response_code = 500
uri = /omicron-zones
|
| 1080 | 2024-06-20T21:03:06.012Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "272a606c-16df-421b-9f4b-b84df51981f3", "content-length": "124", "date": "Thu, 20 Jun 2024 21:03:06 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "272a606c-16df-421b-9f4b-b84df51981f3" }
file = sled-agent/src/rack_setup/service.rs:469
retry_after = 111.984799ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1081 | 2024-06-20T21:03:06.126Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4e960e23-20ba-48e5-9697-e299814c1217, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ef10530a-0825-4479-8014-6a7d3078262d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 25b89476-359e-47f8-bea2-3de9e5e99194, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 271e109e-182e-4626-b37a-fed0542ab9a5, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 70ea324e-1d3b-451f-92e2-3dd4893ba7df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: fe5b0415-63cc-4561-8e18-8b38bd5aa235, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 646a904f-1208-4787-8b83-6ad524b58ce3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } } } }, OmicronZoneConfig { id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } } } }, OmicronZoneConfig { id: bc18ef60-2412-4f2c-9cab-691f92a94385, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1082 | 2024-06-20T21:03:06.126Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1083 | 2024-06-20T21:03:06.126Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1084 | 2024-06-20T21:03:06.149Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1085 | 2024-06-20T21:03:06.149Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1086 | 2024-06-20T21:03:06.169Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1087 | 2024-06-20T21:03:06.169Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1088 | 2024-06-20T21:03:06.190Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1089 | 2024-06-20T21:03:06.190Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1090 | 2024-06-20T21:03:06.211Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1091 | 2024-06-20T21:03:06.211Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1092 | 2024-06-20T21:03:06.232Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1093 | 2024-06-20T21:03:06.232Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1094 | 2024-06-20T21:03:06.252Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1095 | 2024-06-20T21:03:06.253Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1096 | 2024-06-20T21:03:06.272Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1097 | 2024-06-20T21:03:06.272Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1098 | 2024-06-20T21:03:06.291Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1099 | 2024-06-20T21:03:06.291Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1100 | 2024-06-20T21:03:06.312Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1101 | 2024-06-20T21:03:06.312Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1102 | 2024-06-20T21:03:06.332Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1103 | 2024-06-20T21:03:06.332Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1104 | 2024-06-20T21:03:06.351Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1105 | 2024-06-20T21:03:06.351Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1106 | 2024-06-20T21:03:06.372Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1107 | 2024-06-20T21:03:06.372Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1108 | 2024-06-20T21:03:06.391Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1109 | 2024-06-20T21:03:06.391Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1110 | 2024-06-20T21:03:06.410Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1111 | 2024-06-20T21:03:06.410Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1112 | 2024-06-20T21:03:06.429Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1113 | 2024-06-20T21:03:06.429Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1114 | 2024-06-20T21:03:06.448Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1115 | 2024-06-20T21:03:06.448Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1116 | 2024-06-20T21:03:06.467Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1117 | 2024-06-20T21:03:06.467Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1118 | 2024-06-20T21:03:06.486Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1119 | 2024-06-20T21:03:06.486Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1120 | 2024-06-20T21:03:06.508Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1121 | 2024-06-20T21:03:06.508Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1122 | 2024-06-20T21:03:06.528Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1123 | 2024-06-20T21:03:06.528Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1124 | 2024-06-20T21:03:06.528Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1125 | 2024-06-20T21:03:06.528Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1126 | 2024-06-20T21:03:06.620Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 1127 | 2024-06-20T21:03:06.620Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 1128 | 2024-06-20T21:03:06.620Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1129 | 2024-06-20T21:03:06.620Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1130 | 2024-06-20T21:03:06.620Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1131 | 2024-06-20T21:03:06.620Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 1132 | 2024-06-20T21:03:06.620Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 1133 | 2024-06-20T21:03:06.620Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1134 | 2024-06-20T21:03:06.620Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 1135 | 2024-06-20T21:03:06.620Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1136 | 2024-06-20T21:03:06.620Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 1137 | 2024-06-20T21:03:06.620Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1138 | 2024-06-20T21:03:06.620Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 1139 | 2024-06-20T21:03:06.620Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1140 | 2024-06-20T21:03:06.621Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1141 | 2024-06-20T21:03:06.621Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1142 | 2024-06-20T21:03:06.621Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1143 | 2024-06-20T21:03:06.621Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1144 | 2024-06-20T21:03:06.621Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1145 | 2024-06-20T21:03:06.621Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217", TimeNotSynchronized), ("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958", TimeNotSynchronized), ("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235", TimeNotSynchronized), ("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d", TimeNotSynchronized), ("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc", TimeNotSynchronized), ("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57", TimeNotSynchronized), ("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9", TimeNotSynchronized), ("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9", TimeNotSynchronized), ("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194", TimeNotSynchronized), ("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4", TimeNotSynchronized), ("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f", TimeNotSynchronized), ("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc", TimeNotSynchronized), ("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df", TimeNotSynchronized), ("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5", TimeNotSynchronized), ("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec", TimeNotSynchronized), ("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3", TimeNotSynchronized), ("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385", TimeNotSynchronized), ("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4", TimeNotSynchronized), ("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 494952
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:63067
req_id = 20f6e700-e648-4ab2-a96e-25dfdf0e8698
response_code = 500
uri = /omicron-zones
|
| 1146 | 2024-06-20T21:03:06.621Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "20f6e700-e648-4ab2-a96e-25dfdf0e8698", "content-length": "124", "date": "Thu, 20 Jun 2024 21:03:06 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "20f6e700-e648-4ab2-a96e-25dfdf0e8698" }
file = sled-agent/src/rack_setup/service.rs:469
retry_after = 178.114432ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1147 | 2024-06-20T21:03:06.800Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4e960e23-20ba-48e5-9697-e299814c1217, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ef10530a-0825-4479-8014-6a7d3078262d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 25b89476-359e-47f8-bea2-3de9e5e99194, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 271e109e-182e-4626-b37a-fed0542ab9a5, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 70ea324e-1d3b-451f-92e2-3dd4893ba7df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: fe5b0415-63cc-4561-8e18-8b38bd5aa235, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 646a904f-1208-4787-8b83-6ad524b58ce3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } } } }, OmicronZoneConfig { id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } } } }, OmicronZoneConfig { id: bc18ef60-2412-4f2c-9cab-691f92a94385, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1148 | 2024-06-20T21:03:06.801Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1149 | 2024-06-20T21:03:06.801Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1150 | 2024-06-20T21:03:06.824Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1151 | 2024-06-20T21:03:06.824Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1152 | 2024-06-20T21:03:06.846Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1153 | 2024-06-20T21:03:06.846Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1154 | 2024-06-20T21:03:06.866Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1155 | 2024-06-20T21:03:06.866Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1156 | 2024-06-20T21:03:06.886Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1157 | 2024-06-20T21:03:06.886Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1158 | 2024-06-20T21:03:06.904Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1159 | 2024-06-20T21:03:06.904Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1160 | 2024-06-20T21:03:06.925Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1161 | 2024-06-20T21:03:06.925Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1162 | 2024-06-20T21:03:06.945Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1163 | 2024-06-20T21:03:06.945Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1164 | 2024-06-20T21:03:06.964Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1165 | 2024-06-20T21:03:06.964Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1166 | 2024-06-20T21:03:06.985Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1167 | 2024-06-20T21:03:06.985Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1168 | 2024-06-20T21:03:07.006Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1169 | 2024-06-20T21:03:07.006Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1170 | 2024-06-20T21:03:07.025Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1171 | 2024-06-20T21:03:07.025Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1172 | 2024-06-20T21:03:07.045Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1173 | 2024-06-20T21:03:07.045Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1174 | 2024-06-20T21:03:07.066Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1175 | 2024-06-20T21:03:07.066Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1176 | 2024-06-20T21:03:07.086Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1177 | 2024-06-20T21:03:07.086Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1178 | 2024-06-20T21:03:07.105Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1179 | 2024-06-20T21:03:07.105Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1180 | 2024-06-20T21:03:07.125Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1181 | 2024-06-20T21:03:07.125Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1182 | 2024-06-20T21:03:07.145Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1183 | 2024-06-20T21:03:07.145Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1184 | 2024-06-20T21:03:07.166Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1185 | 2024-06-20T21:03:07.167Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1186 | 2024-06-20T21:03:07.188Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1187 | 2024-06-20T21:03:07.188Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1188 | 2024-06-20T21:03:07.208Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1189 | 2024-06-20T21:03:07.208Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1190 | 2024-06-20T21:03:07.208Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1191 | 2024-06-20T21:03:07.208Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1192 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 1193 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1194 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1195 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1196 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1197 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 1198 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 1199 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1200 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1201 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1202 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1203 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1204 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 1205 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1206 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 1207 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1208 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 1209 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 1210 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1211 | 2024-06-20T21:03:07.303Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194", TimeNotSynchronized), ("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235", TimeNotSynchronized), ("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5", TimeNotSynchronized), ("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d", TimeNotSynchronized), ("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc", TimeNotSynchronized), ("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f", TimeNotSynchronized), ("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57", TimeNotSynchronized), ("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec", TimeNotSynchronized), ("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3", TimeNotSynchronized), ("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4", TimeNotSynchronized), ("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0", TimeNotSynchronized), ("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385", TimeNotSynchronized), ("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9", TimeNotSynchronized), ("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc", TimeNotSynchronized), ("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df", TimeNotSynchronized), ("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4", TimeNotSynchronized), ("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958", TimeNotSynchronized), ("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217", TimeNotSynchronized), ("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 502746
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:63067
req_id = 1de8e4de-240c-4b6c-b90e-04832affeb28
response_code = 500
uri = /omicron-zones
|
| 1212 | 2024-06-20T21:03:07.303Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "1de8e4de-240c-4b6c-b90e-04832affeb28", "content-length": "124", "date": "Thu, 20 Jun 2024 21:03:07 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "1de8e4de-240c-4b6c-b90e-04832affeb28" }
file = sled-agent/src/rack_setup/service.rs:469
retry_after = 254.231994ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1213 | 2024-06-20T21:03:07.559Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4e960e23-20ba-48e5-9697-e299814c1217, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ef10530a-0825-4479-8014-6a7d3078262d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 25b89476-359e-47f8-bea2-3de9e5e99194, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 271e109e-182e-4626-b37a-fed0542ab9a5, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 70ea324e-1d3b-451f-92e2-3dd4893ba7df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: fe5b0415-63cc-4561-8e18-8b38bd5aa235, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 646a904f-1208-4787-8b83-6ad524b58ce3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } } } }, OmicronZoneConfig { id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } } } }, OmicronZoneConfig { id: bc18ef60-2412-4f2c-9cab-691f92a94385, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1214 | 2024-06-20T21:03:07.559Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1215 | 2024-06-20T21:03:07.559Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1216 | 2024-06-20T21:03:07.581Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1217 | 2024-06-20T21:03:07.581Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1218 | 2024-06-20T21:03:07.601Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1219 | 2024-06-20T21:03:07.602Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1220 | 2024-06-20T21:03:07.620Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1221 | 2024-06-20T21:03:07.620Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1222 | 2024-06-20T21:03:07.640Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1223 | 2024-06-20T21:03:07.640Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1224 | 2024-06-20T21:03:07.660Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1225 | 2024-06-20T21:03:07.660Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1226 | 2024-06-20T21:03:07.680Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1227 | 2024-06-20T21:03:07.680Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1228 | 2024-06-20T21:03:07.699Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1229 | 2024-06-20T21:03:07.699Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1230 | 2024-06-20T21:03:07.718Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1231 | 2024-06-20T21:03:07.718Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1232 | 2024-06-20T21:03:07.738Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1233 | 2024-06-20T21:03:07.738Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1234 | 2024-06-20T21:03:07.757Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1235 | 2024-06-20T21:03:07.757Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1236 | 2024-06-20T21:03:07.776Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1237 | 2024-06-20T21:03:07.776Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1238 | 2024-06-20T21:03:07.796Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1239 | 2024-06-20T21:03:07.796Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1240 | 2024-06-20T21:03:07.817Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1241 | 2024-06-20T21:03:07.817Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1242 | 2024-06-20T21:03:07.841Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1243 | 2024-06-20T21:03:07.841Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1244 | 2024-06-20T21:03:07.859Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1245 | 2024-06-20T21:03:07.859Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1246 | 2024-06-20T21:03:07.879Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1247 | 2024-06-20T21:03:07.880Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1248 | 2024-06-20T21:03:07.897Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1249 | 2024-06-20T21:03:07.897Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1250 | 2024-06-20T21:03:07.918Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1251 | 2024-06-20T21:03:07.918Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1252 | 2024-06-20T21:03:07.936Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1253 | 2024-06-20T21:03:07.937Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1254 | 2024-06-20T21:03:07.958Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1255 | 2024-06-20T21:03:07.958Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1256 | 2024-06-20T21:03:07.958Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1257 | 2024-06-20T21:03:07.958Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1258 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1259 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1260 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1261 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 1262 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 1263 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 1264 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1265 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1266 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1267 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 1268 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1269 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 1270 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1271 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1272 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1273 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1274 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1275 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 1276 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 1277 | 2024-06-20T21:03:08.056Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec", TimeNotSynchronized), ("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0", TimeNotSynchronized), ("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235", TimeNotSynchronized), ("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217", TimeNotSynchronized), ("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194", TimeNotSynchronized), ("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958", TimeNotSynchronized), ("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3", TimeNotSynchronized), ("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d", TimeNotSynchronized), ("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385", TimeNotSynchronized), ("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df", TimeNotSynchronized), ("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4", TimeNotSynchronized), ("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9", TimeNotSynchronized), ("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5", TimeNotSynchronized), ("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc", TimeNotSynchronized), ("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4", TimeNotSynchronized), ("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc", TimeNotSynchronized), ("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9", TimeNotSynchronized), ("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57", TimeNotSynchronized), ("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 497208
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:63067
req_id = fd6bf899-8c33-4973-8a33-d95aede3f453
response_code = 500
uri = /omicron-zones
|
| 1278 | 2024-06-20T21:03:08.056Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "fd6bf899-8c33-4973-8a33-d95aede3f453", "content-length": "124", "date": "Thu, 20 Jun 2024 21:03:07 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "fd6bf899-8c33-4973-8a33-d95aede3f453" }
file = sled-agent/src/rack_setup/service.rs:469
retry_after = 373.094304ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1279 | 2024-06-20T21:03:08.431Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4e960e23-20ba-48e5-9697-e299814c1217, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ef10530a-0825-4479-8014-6a7d3078262d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 25b89476-359e-47f8-bea2-3de9e5e99194, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 271e109e-182e-4626-b37a-fed0542ab9a5, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 70ea324e-1d3b-451f-92e2-3dd4893ba7df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: fe5b0415-63cc-4561-8e18-8b38bd5aa235, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 646a904f-1208-4787-8b83-6ad524b58ce3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } } } }, OmicronZoneConfig { id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } } } }, OmicronZoneConfig { id: bc18ef60-2412-4f2c-9cab-691f92a94385, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1280 | 2024-06-20T21:03:08.431Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1281 | 2024-06-20T21:03:08.431Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1282 | 2024-06-20T21:03:08.453Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1283 | 2024-06-20T21:03:08.453Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1284 | 2024-06-20T21:03:08.470Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1285 | 2024-06-20T21:03:08.470Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1286 | 2024-06-20T21:03:08.491Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1287 | 2024-06-20T21:03:08.491Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1288 | 2024-06-20T21:03:08.509Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1289 | 2024-06-20T21:03:08.509Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1290 | 2024-06-20T21:03:08.527Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1291 | 2024-06-20T21:03:08.527Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1292 | 2024-06-20T21:03:08.545Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1293 | 2024-06-20T21:03:08.545Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1294 | 2024-06-20T21:03:08.564Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1295 | 2024-06-20T21:03:08.564Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1296 | 2024-06-20T21:03:08.582Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1297 | 2024-06-20T21:03:08.582Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1298 | 2024-06-20T21:03:08.601Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1299 | 2024-06-20T21:03:08.601Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1300 | 2024-06-20T21:03:08.620Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1301 | 2024-06-20T21:03:08.620Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1302 | 2024-06-20T21:03:08.642Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1303 | 2024-06-20T21:03:08.642Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1304 | 2024-06-20T21:03:08.664Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1305 | 2024-06-20T21:03:08.664Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1306 | 2024-06-20T21:03:08.683Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1307 | 2024-06-20T21:03:08.683Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1308 | 2024-06-20T21:03:08.704Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1309 | 2024-06-20T21:03:08.704Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1310 | 2024-06-20T21:03:08.724Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1311 | 2024-06-20T21:03:08.724Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1312 | 2024-06-20T21:03:08.744Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1313 | 2024-06-20T21:03:08.744Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1314 | 2024-06-20T21:03:08.762Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1315 | 2024-06-20T21:03:08.762Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1316 | 2024-06-20T21:03:08.781Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1317 | 2024-06-20T21:03:08.781Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1318 | 2024-06-20T21:03:08.800Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1319 | 2024-06-20T21:03:08.800Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1320 | 2024-06-20T21:03:08.821Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1321 | 2024-06-20T21:03:08.821Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1322 | 2024-06-20T21:03:08.822Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1323 | 2024-06-20T21:03:08.822Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1324 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1325 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 1326 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1327 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1328 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1329 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1330 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1331 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 1332 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1333 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 1334 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 1335 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 1336 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 1337 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1338 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1339 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1340 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1341 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 1342 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1343 | 2024-06-20T21:03:08.916Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385", TimeNotSynchronized), ("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df", TimeNotSynchronized), ("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3", TimeNotSynchronized), ("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0", TimeNotSynchronized), ("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4", TimeNotSynchronized), ("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec", TimeNotSynchronized), ("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d", TimeNotSynchronized), ("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9", TimeNotSynchronized), ("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5", TimeNotSynchronized), ("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57", TimeNotSynchronized), ("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958", TimeNotSynchronized), ("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217", TimeNotSynchronized), ("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194", TimeNotSynchronized), ("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235", TimeNotSynchronized), ("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc", TimeNotSynchronized), ("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9", TimeNotSynchronized), ("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4", TimeNotSynchronized), ("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f", TimeNotSynchronized), ("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 484808
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:63067
req_id = 8f0cd56e-026f-4b58-9f7b-30d06950c479
response_code = 500
uri = /omicron-zones
|
| 1344 | 2024-06-20T21:03:08.916Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "8f0cd56e-026f-4b58-9f7b-30d06950c479", "content-length": "124", "date": "Thu, 20 Jun 2024 21:03:08 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "8f0cd56e-026f-4b58-9f7b-30d06950c479" }
file = sled-agent/src/rack_setup/service.rs:469
retry_after = 472.147939ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1345 | 2024-06-20T21:03:09.389Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4e960e23-20ba-48e5-9697-e299814c1217, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ef10530a-0825-4479-8014-6a7d3078262d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 25b89476-359e-47f8-bea2-3de9e5e99194, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 271e109e-182e-4626-b37a-fed0542ab9a5, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 70ea324e-1d3b-451f-92e2-3dd4893ba7df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: fe5b0415-63cc-4561-8e18-8b38bd5aa235, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 646a904f-1208-4787-8b83-6ad524b58ce3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } } } }, OmicronZoneConfig { id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } } } }, OmicronZoneConfig { id: bc18ef60-2412-4f2c-9cab-691f92a94385, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1346 | 2024-06-20T21:03:09.390Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1347 | 2024-06-20T21:03:09.390Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1348 | 2024-06-20T21:03:09.411Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1349 | 2024-06-20T21:03:09.411Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1350 | 2024-06-20T21:03:09.430Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1351 | 2024-06-20T21:03:09.430Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1352 | 2024-06-20T21:03:09.448Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1353 | 2024-06-20T21:03:09.448Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1354 | 2024-06-20T21:03:09.466Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1355 | 2024-06-20T21:03:09.466Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1356 | 2024-06-20T21:03:09.486Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1357 | 2024-06-20T21:03:09.486Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1358 | 2024-06-20T21:03:09.510Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1359 | 2024-06-20T21:03:09.510Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1360 | 2024-06-20T21:03:09.537Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1361 | 2024-06-20T21:03:09.537Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1362 | 2024-06-20T21:03:09.565Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1363 | 2024-06-20T21:03:09.565Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1364 | 2024-06-20T21:03:09.592Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1365 | 2024-06-20T21:03:09.592Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1366 | 2024-06-20T21:03:09.618Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1367 | 2024-06-20T21:03:09.618Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1368 | 2024-06-20T21:03:09.643Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1369 | 2024-06-20T21:03:09.643Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1370 | 2024-06-20T21:03:09.669Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1371 | 2024-06-20T21:03:09.669Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1372 | 2024-06-20T21:03:09.694Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1373 | 2024-06-20T21:03:09.694Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1374 | 2024-06-20T21:03:09.717Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1375 | 2024-06-20T21:03:09.717Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1376 | 2024-06-20T21:03:09.741Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1377 | 2024-06-20T21:03:09.741Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1378 | 2024-06-20T21:03:09.765Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1379 | 2024-06-20T21:03:09.765Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1380 | 2024-06-20T21:03:09.788Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1381 | 2024-06-20T21:03:09.788Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1382 | 2024-06-20T21:03:09.810Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1383 | 2024-06-20T21:03:09.810Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1384 | 2024-06-20T21:03:09.832Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1385 | 2024-06-20T21:03:09.832Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1386 | 2024-06-20T21:03:09.859Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1387 | 2024-06-20T21:03:09.859Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1388 | 2024-06-20T21:03:09.859Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1389 | 2024-06-20T21:03:09.859Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1390 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1391 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1392 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 1393 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 1394 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 1395 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 1396 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1397 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1398 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1399 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1400 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1401 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 1402 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1403 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1404 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 1405 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1406 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 1407 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1408 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1409 | 2024-06-20T21:03:09.965Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc", TimeNotSynchronized), ("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4", TimeNotSynchronized), ("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958", TimeNotSynchronized), ("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9", TimeNotSynchronized), ("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57", TimeNotSynchronized), ("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217", TimeNotSynchronized), ("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d", TimeNotSynchronized), ("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235", TimeNotSynchronized), ("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc", TimeNotSynchronized), ("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5", TimeNotSynchronized), ("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3", TimeNotSynchronized), ("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194", TimeNotSynchronized), ("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4", TimeNotSynchronized), ("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0", TimeNotSynchronized), ("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f", TimeNotSynchronized), ("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec", TimeNotSynchronized), ("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df", TimeNotSynchronized), ("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385", TimeNotSynchronized), ("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 575194
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:63067
req_id = 01b2be0d-7218-434d-b8c4-226a706851d0
response_code = 500
uri = /omicron-zones
|
| 1410 | 2024-06-20T21:03:09.965Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "01b2be0d-7218-434d-b8c4-226a706851d0", "content-length": "124", "date": "Thu, 20 Jun 2024 21:03:09 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "01b2be0d-7218-434d-b8c4-226a706851d0" }
file = sled-agent/src/rack_setup/service.rs:469
retry_after = 413.422463ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1411 | 2024-06-20T21:03:10.380Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4e960e23-20ba-48e5-9697-e299814c1217, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ef10530a-0825-4479-8014-6a7d3078262d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 25b89476-359e-47f8-bea2-3de9e5e99194, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 271e109e-182e-4626-b37a-fed0542ab9a5, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 70ea324e-1d3b-451f-92e2-3dd4893ba7df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: fe5b0415-63cc-4561-8e18-8b38bd5aa235, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 646a904f-1208-4787-8b83-6ad524b58ce3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } } } }, OmicronZoneConfig { id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } } } }, OmicronZoneConfig { id: bc18ef60-2412-4f2c-9cab-691f92a94385, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1412 | 2024-06-20T21:03:10.380Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1413 | 2024-06-20T21:03:10.380Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1414 | 2024-06-20T21:03:10.402Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1415 | 2024-06-20T21:03:10.402Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1416 | 2024-06-20T21:03:10.423Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1417 | 2024-06-20T21:03:10.423Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1418 | 2024-06-20T21:03:10.441Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1419 | 2024-06-20T21:03:10.441Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1420 | 2024-06-20T21:03:10.459Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1421 | 2024-06-20T21:03:10.459Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1422 | 2024-06-20T21:03:10.478Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1423 | 2024-06-20T21:03:10.478Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1424 | 2024-06-20T21:03:10.496Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1425 | 2024-06-20T21:03:10.496Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1426 | 2024-06-20T21:03:10.517Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1427 | 2024-06-20T21:03:10.517Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1428 | 2024-06-20T21:03:10.537Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1429 | 2024-06-20T21:03:10.537Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1430 | 2024-06-20T21:03:10.557Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1431 | 2024-06-20T21:03:10.557Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1432 | 2024-06-20T21:03:10.576Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1433 | 2024-06-20T21:03:10.576Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1434 | 2024-06-20T21:03:10.594Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1435 | 2024-06-20T21:03:10.594Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1436 | 2024-06-20T21:03:10.614Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1437 | 2024-06-20T21:03:10.614Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1438 | 2024-06-20T21:03:10.633Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1439 | 2024-06-20T21:03:10.633Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1440 | 2024-06-20T21:03:10.652Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1441 | 2024-06-20T21:03:10.652Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1442 | 2024-06-20T21:03:10.670Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1443 | 2024-06-20T21:03:10.670Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1444 | 2024-06-20T21:03:10.690Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1445 | 2024-06-20T21:03:10.690Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1446 | 2024-06-20T21:03:10.708Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1447 | 2024-06-20T21:03:10.708Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1448 | 2024-06-20T21:03:10.727Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1449 | 2024-06-20T21:03:10.727Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1450 | 2024-06-20T21:03:10.745Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1451 | 2024-06-20T21:03:10.746Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1452 | 2024-06-20T21:03:10.764Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1453 | 2024-06-20T21:03:10.764Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1454 | 2024-06-20T21:03:10.764Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1455 | 2024-06-20T21:03:10.765Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1456 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1457 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1458 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1459 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1460 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1461 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 1462 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1463 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 1464 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 1465 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 1466 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1467 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1468 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1469 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 1470 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1471 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 1472 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 1473 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1474 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1475 | 2024-06-20T21:03:10.859Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3", TimeNotSynchronized), ("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0", TimeNotSynchronized), ("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385", TimeNotSynchronized), ("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9", TimeNotSynchronized), ("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5", TimeNotSynchronized), ("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df", TimeNotSynchronized), ("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc", TimeNotSynchronized), ("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f", TimeNotSynchronized), ("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9", TimeNotSynchronized), ("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217", TimeNotSynchronized), ("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec", TimeNotSynchronized), ("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4", TimeNotSynchronized), ("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235", TimeNotSynchronized), ("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958", TimeNotSynchronized), ("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d", TimeNotSynchronized), ("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57", TimeNotSynchronized), ("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194", TimeNotSynchronized), ("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4", TimeNotSynchronized), ("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 479263
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:63067
req_id = 69d4ba2e-82e5-4510-a40c-3e9e9a8c1ba7
response_code = 500
uri = /omicron-zones
|
| 1476 | 2024-06-20T21:03:10.859Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "69d4ba2e-82e5-4510-a40c-3e9e9a8c1ba7", "content-length": "124", "date": "Thu, 20 Jun 2024 21:03:10 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "69d4ba2e-82e5-4510-a40c-3e9e9a8c1ba7" }
file = sled-agent/src/rack_setup/service.rs:469
retry_after = 495.517727ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1477 | 2024-06-20T21:03:11.356Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4e960e23-20ba-48e5-9697-e299814c1217, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ef10530a-0825-4479-8014-6a7d3078262d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 25b89476-359e-47f8-bea2-3de9e5e99194, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 271e109e-182e-4626-b37a-fed0542ab9a5, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 70ea324e-1d3b-451f-92e2-3dd4893ba7df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: fe5b0415-63cc-4561-8e18-8b38bd5aa235, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 646a904f-1208-4787-8b83-6ad524b58ce3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } } } }, OmicronZoneConfig { id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } } } }, OmicronZoneConfig { id: bc18ef60-2412-4f2c-9cab-691f92a94385, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1478 | 2024-06-20T21:03:11.356Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1479 | 2024-06-20T21:03:11.356Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1480 | 2024-06-20T21:03:11.378Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1481 | 2024-06-20T21:03:11.378Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1482 | 2024-06-20T21:03:11.397Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1483 | 2024-06-20T21:03:11.397Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1484 | 2024-06-20T21:03:11.418Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1485 | 2024-06-20T21:03:11.418Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1486 | 2024-06-20T21:03:11.437Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1487 | 2024-06-20T21:03:11.437Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1488 | 2024-06-20T21:03:11.455Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1489 | 2024-06-20T21:03:11.455Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1490 | 2024-06-20T21:03:11.475Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1491 | 2024-06-20T21:03:11.475Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1492 | 2024-06-20T21:03:11.494Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1493 | 2024-06-20T21:03:11.494Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1494 | 2024-06-20T21:03:11.516Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1495 | 2024-06-20T21:03:11.516Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1496 | 2024-06-20T21:03:11.536Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1497 | 2024-06-20T21:03:11.536Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1498 | 2024-06-20T21:03:11.556Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1499 | 2024-06-20T21:03:11.556Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1500 | 2024-06-20T21:03:11.577Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1501 | 2024-06-20T21:03:11.577Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1502 | 2024-06-20T21:03:11.596Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1503 | 2024-06-20T21:03:11.596Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1504 | 2024-06-20T21:03:11.615Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1505 | 2024-06-20T21:03:11.615Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1506 | 2024-06-20T21:03:11.636Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1507 | 2024-06-20T21:03:11.636Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1508 | 2024-06-20T21:03:11.655Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1509 | 2024-06-20T21:03:11.655Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1510 | 2024-06-20T21:03:11.674Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1511 | 2024-06-20T21:03:11.674Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1512 | 2024-06-20T21:03:11.694Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1513 | 2024-06-20T21:03:11.694Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1514 | 2024-06-20T21:03:11.714Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1515 | 2024-06-20T21:03:11.714Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1516 | 2024-06-20T21:03:11.735Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1517 | 2024-06-20T21:03:11.735Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1518 | 2024-06-20T21:03:11.754Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1519 | 2024-06-20T21:03:11.754Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1520 | 2024-06-20T21:03:11.754Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1521 | 2024-06-20T21:03:11.754Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1522 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 1523 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1524 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1525 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1526 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1527 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 1528 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1529 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1530 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1531 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1532 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 1533 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1534 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1535 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 1536 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 1537 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1538 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1539 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 1540 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 1541 | 2024-06-20T21:03:11.850Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217", TimeNotSynchronized), ("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9", TimeNotSynchronized), ("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3", TimeNotSynchronized), ("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0", TimeNotSynchronized), ("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d", TimeNotSynchronized), ("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df", TimeNotSynchronized), ("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec", TimeNotSynchronized), ("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4", TimeNotSynchronized), ("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5", TimeNotSynchronized), ("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc", TimeNotSynchronized), ("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194", TimeNotSynchronized), ("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc", TimeNotSynchronized), ("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4", TimeNotSynchronized), ("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9", TimeNotSynchronized), ("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f", TimeNotSynchronized), ("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385", TimeNotSynchronized), ("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235", TimeNotSynchronized), ("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958", TimeNotSynchronized), ("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 493787
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:63067
req_id = bcb67277-eb74-43fa-b662-d9a66fabadf7
response_code = 500
uri = /omicron-zones
|
| 1542 | 2024-06-20T21:03:11.850Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "bcb67277-eb74-43fa-b662-d9a66fabadf7", "content-length": "124", "date": "Thu, 20 Jun 2024 21:03:11 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "bcb67277-eb74-43fa-b662-d9a66fabadf7" }
file = sled-agent/src/rack_setup/service.rs:469
retry_after = 676.494237ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1543 | 2024-06-20T21:03:12.528Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4e960e23-20ba-48e5-9697-e299814c1217, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ef10530a-0825-4479-8014-6a7d3078262d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 25b89476-359e-47f8-bea2-3de9e5e99194, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 271e109e-182e-4626-b37a-fed0542ab9a5, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 70ea324e-1d3b-451f-92e2-3dd4893ba7df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: fe5b0415-63cc-4561-8e18-8b38bd5aa235, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 646a904f-1208-4787-8b83-6ad524b58ce3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } } } }, OmicronZoneConfig { id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } } } }, OmicronZoneConfig { id: bc18ef60-2412-4f2c-9cab-691f92a94385, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1544 | 2024-06-20T21:03:12.528Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1545 | 2024-06-20T21:03:12.528Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1546 | 2024-06-20T21:03:12.550Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1547 | 2024-06-20T21:03:12.550Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1548 | 2024-06-20T21:03:12.569Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1549 | 2024-06-20T21:03:12.570Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1550 | 2024-06-20T21:03:12.588Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1551 | 2024-06-20T21:03:12.588Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1552 | 2024-06-20T21:03:12.608Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1553 | 2024-06-20T21:03:12.608Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1554 | 2024-06-20T21:03:12.626Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1555 | 2024-06-20T21:03:12.626Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1556 | 2024-06-20T21:03:12.644Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1557 | 2024-06-20T21:03:12.644Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1558 | 2024-06-20T21:03:12.663Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1559 | 2024-06-20T21:03:12.663Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1560 | 2024-06-20T21:03:12.681Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1561 | 2024-06-20T21:03:12.681Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1562 | 2024-06-20T21:03:12.699Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1563 | 2024-06-20T21:03:12.699Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1564 | 2024-06-20T21:03:12.719Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1565 | 2024-06-20T21:03:12.719Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1566 | 2024-06-20T21:03:12.739Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1567 | 2024-06-20T21:03:12.739Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1568 | 2024-06-20T21:03:12.759Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1569 | 2024-06-20T21:03:12.759Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1570 | 2024-06-20T21:03:12.779Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1571 | 2024-06-20T21:03:12.779Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1572 | 2024-06-20T21:03:12.797Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1573 | 2024-06-20T21:03:12.797Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1574 | 2024-06-20T21:03:12.816Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1575 | 2024-06-20T21:03:12.816Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1576 | 2024-06-20T21:03:12.834Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1577 | 2024-06-20T21:03:12.834Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1578 | 2024-06-20T21:03:12.851Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1579 | 2024-06-20T21:03:12.851Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1580 | 2024-06-20T21:03:12.870Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1581 | 2024-06-20T21:03:12.871Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1582 | 2024-06-20T21:03:12.891Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1583 | 2024-06-20T21:03:12.891Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1584 | 2024-06-20T21:03:12.911Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1585 | 2024-06-20T21:03:12.911Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1586 | 2024-06-20T21:03:12.912Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1587 | 2024-06-20T21:03:12.912Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1588 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1589 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 1590 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1591 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1592 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1593 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 1594 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1595 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 1596 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1597 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1598 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 1599 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1600 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1601 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1602 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1603 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 1604 | 2024-06-20T21:03:13.005Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 1605 | 2024-06-20T21:03:13.006Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 1606 | 2024-06-20T21:03:13.006Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1607 | 2024-06-20T21:03:13.006Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc", TimeNotSynchronized), ("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9", TimeNotSynchronized), ("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d", TimeNotSynchronized), ("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc", TimeNotSynchronized), ("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385", TimeNotSynchronized), ("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958", TimeNotSynchronized), ("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235", TimeNotSynchronized), ("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57", TimeNotSynchronized), ("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4", TimeNotSynchronized), ("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5", TimeNotSynchronized), ("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f", TimeNotSynchronized), ("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec", TimeNotSynchronized), ("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9", TimeNotSynchronized), ("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3", TimeNotSynchronized), ("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0", TimeNotSynchronized), ("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217", TimeNotSynchronized), ("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194", TimeNotSynchronized), ("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df", TimeNotSynchronized), ("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 477952
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:63067
req_id = 21220cd4-0449-42fa-990b-7c6a3679e49c
response_code = 500
uri = /omicron-zones
|
| 1608 | 2024-06-20T21:03:13.006Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "21220cd4-0449-42fa-990b-7c6a3679e49c", "content-length": "124", "date": "Thu, 20 Jun 2024 21:03:13 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "21220cd4-0449-42fa-990b-7c6a3679e49c" }
file = sled-agent/src/rack_setup/service.rs:469
retry_after = 459.643939ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1609 | 2024-06-20T21:03:13.466Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4e960e23-20ba-48e5-9697-e299814c1217, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ef10530a-0825-4479-8014-6a7d3078262d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 25b89476-359e-47f8-bea2-3de9e5e99194, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 271e109e-182e-4626-b37a-fed0542ab9a5, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 70ea324e-1d3b-451f-92e2-3dd4893ba7df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: fe5b0415-63cc-4561-8e18-8b38bd5aa235, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 646a904f-1208-4787-8b83-6ad524b58ce3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } } } }, OmicronZoneConfig { id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } } } }, OmicronZoneConfig { id: bc18ef60-2412-4f2c-9cab-691f92a94385, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1610 | 2024-06-20T21:03:13.467Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1611 | 2024-06-20T21:03:13.467Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1612 | 2024-06-20T21:03:13.487Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1613 | 2024-06-20T21:03:13.487Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1614 | 2024-06-20T21:03:13.507Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1615 | 2024-06-20T21:03:13.507Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1616 | 2024-06-20T21:03:13.525Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1617 | 2024-06-20T21:03:13.525Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1618 | 2024-06-20T21:03:13.545Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1619 | 2024-06-20T21:03:13.545Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1620 | 2024-06-20T21:03:13.564Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1621 | 2024-06-20T21:03:13.564Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1622 | 2024-06-20T21:03:13.582Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1623 | 2024-06-20T21:03:13.582Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1624 | 2024-06-20T21:03:13.600Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1625 | 2024-06-20T21:03:13.600Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1626 | 2024-06-20T21:03:13.618Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1627 | 2024-06-20T21:03:13.618Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1628 | 2024-06-20T21:03:13.641Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1629 | 2024-06-20T21:03:13.641Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1630 | 2024-06-20T21:03:13.662Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1631 | 2024-06-20T21:03:13.662Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1632 | 2024-06-20T21:03:13.681Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1633 | 2024-06-20T21:03:13.681Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1634 | 2024-06-20T21:03:13.699Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1635 | 2024-06-20T21:03:13.699Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1636 | 2024-06-20T21:03:13.717Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1637 | 2024-06-20T21:03:13.717Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1638 | 2024-06-20T21:03:13.737Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1639 | 2024-06-20T21:03:13.737Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1640 | 2024-06-20T21:03:13.756Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1641 | 2024-06-20T21:03:13.756Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1642 | 2024-06-20T21:03:13.774Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1643 | 2024-06-20T21:03:13.774Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1644 | 2024-06-20T21:03:13.793Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1645 | 2024-06-20T21:03:13.793Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1646 | 2024-06-20T21:03:13.811Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1647 | 2024-06-20T21:03:13.811Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1648 | 2024-06-20T21:03:13.832Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1649 | 2024-06-20T21:03:13.832Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1650 | 2024-06-20T21:03:13.852Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1651 | 2024-06-20T21:03:13.852Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1652 | 2024-06-20T21:03:13.852Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1653 | 2024-06-20T21:03:13.852Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1654 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1655 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 1656 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 1657 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1658 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1659 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1660 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1661 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 1662 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1663 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 1664 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1665 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1666 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 1667 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 1668 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1669 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 1670 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1671 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1672 | 2024-06-20T21:03:13.946Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3060
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1673 | 2024-06-20T21:03:13.946Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3", TimeNotSynchronized), ("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57", TimeNotSynchronized), ("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958", TimeNotSynchronized), ("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5", TimeNotSynchronized), ("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0", TimeNotSynchronized), ("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4", TimeNotSynchronized), ("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9", TimeNotSynchronized), ("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9", TimeNotSynchronized), ("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d", TimeNotSynchronized), ("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df", TimeNotSynchronized), ("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385", TimeNotSynchronized), ("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec", TimeNotSynchronized), ("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f", TimeNotSynchronized), ("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194", TimeNotSynchronized), ("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc", TimeNotSynchronized), ("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217", TimeNotSynchronized), ("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4", TimeNotSynchronized), ("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc", TimeNotSynchronized), ("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866
latency_us = 480041
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:63067
req_id = 628e1e3b-1d77-4e72-9cad-c70d8ab9263c
response_code = 500
uri = /omicron-zones
|
| 1674 | 2024-06-20T21:03:13.947Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "628e1e3b-1d77-4e72-9cad-c70d8ab9263c", "content-length": "124", "date": "Thu, 20 Jun 2024 21:03:13 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "628e1e3b-1d77-4e72-9cad-c70d8ab9263c" }
file = sled-agent/src/rack_setup/service.rs:469
retry_after = 1.253293378s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1675 | 2024-06-20T21:03:15.200Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a23901b9-ec53-4e80-87b3-8a5db532b3da, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: b294fa61-9f44-4568-afb9-45e8dfed810d, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bdbb40ec-3fc3-47e8-9565-b8e1cdc94225, kind: Service { id: 9c3e0325-17c9-41b6-9df0-8049b765cb12 }, name: Name("ntp-9c3e0325-17c9-41b6-9df0-8049b765cb12"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 81])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 541f1717-17b0-4191-be31-eab4c1500570, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 450b3b5f-7293-4a8e-978d-a22e72953896, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: f71b4047-e660-4f04-b1b8-cae0ee49769f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4e960e23-20ba-48e5-9697-e299814c1217, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ef10530a-0825-4479-8014-6a7d3078262d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 25b89476-359e-47f8-bea2-3de9e5e99194, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 271e109e-182e-4626-b37a-fed0542ab9a5, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 70ea324e-1d3b-451f-92e2-3dd4893ba7df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: fe5b0415-63cc-4561-8e18-8b38bd5aa235, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External } } } }, OmicronZoneConfig { id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External } } } }, OmicronZoneConfig { id: 646a904f-1208-4787-8b83-6ad524b58ce3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External } } } }, OmicronZoneConfig { id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External } } } }, OmicronZoneConfig { id: bc18ef60-2412-4f2c-9cab-691f92a94385, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:431
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1676 | 2024-06-20T21:03:15.201Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1677 | 2024-06-20T21:03:15.201Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 8c770d8b-5afa-42fc-ad13-e754ea10ed0b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1678 | 2024-06-20T21:03:15.222Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1679 | 2024-06-20T21:03:15.222Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a23901b9-ec53-4e80-87b3-8a5db532b3da, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1680 | 2024-06-20T21:03:15.240Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1681 | 2024-06-20T21:03:15.240Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b294fa61-9f44-4568-afb9-45e8dfed810d, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1682 | 2024-06-20T21:03:15.260Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1683 | 2024-06-20T21:03:15.260Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 541f1717-17b0-4191-be31-eab4c1500570, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1684 | 2024-06-20T21:03:15.278Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1685 | 2024-06-20T21:03:15.278Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 021a4838-0953-41f3-a0c1-d3ce533ca2ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1686 | 2024-06-20T21:03:15.296Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1687 | 2024-06-20T21:03:15.296Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 450b3b5f-7293-4a8e-978d-a22e72953896, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1688 | 2024-06-20T21:03:15.001Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1689 | 2024-06-20T21:03:15.001Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f71b4047-e660-4f04-b1b8-cae0ee49769f, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1690 | 2024-06-20T21:03:15.020Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1691 | 2024-06-20T21:03:15.020Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9e1c26a-87d3-4501-a588-62f9673bcb7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1692 | 2024-06-20T21:03:15.042Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1693 | 2024-06-20T21:03:15.042Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1694 | 2024-06-20T21:03:15.062Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1695 | 2024-06-20T21:03:15.062Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 4e960e23-20ba-48e5-9697-e299814c1217, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1696 | 2024-06-20T21:03:15.081Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1697 | 2024-06-20T21:03:15.081Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 25b89476-359e-47f8-bea2-3de9e5e99194, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1698 | 2024-06-20T21:03:15.101Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1699 | 2024-06-20T21:03:15.101Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc, dataset_name: DatasetName { pool_name: ZpoolName { id: cc5f6a04-2111-4941-923c-8867d737d5fc (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1700 | 2024-06-20T21:03:15.120Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1701 | 2024-06-20T21:03:15.120Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2d29a557-7bee-4fe1-8b8e-8c4309441fe4, dataset_name: DatasetName { pool_name: ZpoolName { id: 662892f9-e563-4653-bedf-02b82422ec9a (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1702 | 2024-06-20T21:03:15.138Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1703 | 2024-06-20T21:03:15.138Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 3b7765ae-cc70-4a27-9b47-752b8a12d69f, dataset_name: DatasetName { pool_name: ZpoolName { id: f32a5037-29da-4375-a8f1-8e9a8b5d3778 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1704 | 2024-06-20T21:03:15.156Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1705 | 2024-06-20T21:03:15.156Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: b9bdba62-62b5-45f2-8e6d-7415e56d8efc, dataset_name: DatasetName { pool_name: ZpoolName { id: 339b70a8-c294-4120-9bd3-76df19004c3b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1706 | 2024-06-20T21:03:15.174Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1707 | 2024-06-20T21:03:15.174Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2e9691b8-007b-4520-947b-a0537ec7b2ec, dataset_name: DatasetName { pool_name: ZpoolName { id: 63627367-eb03-4016-8c1d-73768441f612 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1708 | 2024-06-20T21:03:15.193Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1709 | 2024-06-20T21:03:15.193Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 646a904f-1208-4787-8b83-6ad524b58ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2e39fd13-5176-4455-b633-7660296b7576 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1710 | 2024-06-20T21:03:15.211Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1711 | 2024-06-20T21:03:15.211Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 634cce40-bafd-4e0d-96ed-afdcb1d112b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ff415f8-ea2c-44b4-b9a7-2a7cf086871c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1712 | 2024-06-20T21:03:15.229Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1713 | 2024-06-20T21:03:15.229Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: bc18ef60-2412-4f2c-9cab-691f92a94385, dataset_name: DatasetName { pool_name: ZpoolName { id: 39d4ca15-26b5-4bd0-bbf6-fd95c2268c88 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1714 | 2024-06-20T21:03:15.248Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1715 | 2024-06-20T21:03:15.248Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 83db2fa4-dc9c-4ccd-a225-33f67333fdb4, dataset_name: DatasetName { pool_name: ZpoolName { id: ac756826-7d23-4a48-8f48-471e91447b90 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1716 | 2024-06-20T21:03:15.266Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1717 | 2024-06-20T21:03:15.266Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1718 | 2024-06-20T21:03:15.266Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1719 | 2024-06-20T21:03:15.266Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1720 | 2024-06-20T21:03:15.376Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:263
|
| 1721 | 2024-06-20T21:03:15.376Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1722 | 2024-06-20T21:03:15.376Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1723 | 2024-06-20T21:03:15.411Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:263
|
| 1724 | 2024-06-20T21:03:15.411Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1725 | 2024-06-20T21:03:15.411Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1726 | 2024-06-20T21:03:15.411Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1727 | 2024-06-20T21:03:15.436Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1728 | 2024-06-20T21:03:15.436Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1729 | 2024-06-20T21:03:15.436Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1730 | 2024-06-20T21:03:15.619Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:263
|
| 1731 | 2024-06-20T21:03:15.620Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1732 | 2024-06-20T21:03:15.620Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1733 | 2024-06-20T21:03:15.620Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1734 | 2024-06-20T21:03:15.632Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:263
|
| 1735 | 2024-06-20T21:03:15.632Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1736 | 2024-06-20T21:03:15.634Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:263
|
| 1737 | 2024-06-20T21:03:15.657Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1738 | 2024-06-20T21:03:15.657Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1739 | 2024-06-20T21:03:15.657Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
file = illumos-utils/src/zone.rs:315
|
| 1740 | 2024-06-20T21:03:15.663Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
file = illumos-utils/src/zone.rs:315
|
| 1741 | 2024-06-20T21:03:15.667Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
file = illumos-utils/src/zone.rs:315
|
| 1742 | 2024-06-20T21:03:15.672Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
file = illumos-utils/src/zone.rs:315
|
| 1743 | 2024-06-20T21:03:15.678Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
file = illumos-utils/src/zone.rs:315
|
| 1744 | 2024-06-20T21:03:15.683Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
file = illumos-utils/src/zone.rs:315
|
| 1745 | 2024-06-20T21:03:15.687Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
file = illumos-utils/src/zone.rs:315
|
| 1746 | 2024-06-20T21:03:15.692Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
file = illumos-utils/src/zone.rs:315
|
| 1747 | 2024-06-20T21:03:15.805Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
file = illumos-utils/src/zone.rs:354
|
| 1748 | 2024-06-20T21:03:15.810Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
file = illumos-utils/src/zone.rs:354
|
| 1749 | 2024-06-20T21:03:15.814Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
file = illumos-utils/src/zone.rs:354
|
| 1750 | 2024-06-20T21:03:15.818Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
file = illumos-utils/src/zone.rs:354
|
| 1751 | 2024-06-20T21:03:15.822Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
file = illumos-utils/src/zone.rs:354
|
| 1752 | 2024-06-20T21:03:15.827Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
file = illumos-utils/src/zone.rs:354
|
| 1753 | 2024-06-20T21:03:15.831Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
file = illumos-utils/src/zone.rs:354
|
| 1754 | 2024-06-20T21:03:15.836Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
file = illumos-utils/src/zone.rs:354
|
| 1755 | 2024-06-20T21:03:15.840Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
file = illumos-utils/src/zone.rs:315
|
| 1756 | 2024-06-20T21:03:15.845Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
file = illumos-utils/src/zone.rs:315
|
| 1757 | 2024-06-20T21:03:15.850Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
file = illumos-utils/src/zone.rs:315
|
| 1758 | 2024-06-20T21:03:15.855Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
file = illumos-utils/src/zone.rs:315
|
| 1759 | 2024-06-20T21:03:15.860Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
file = illumos-utils/src/zone.rs:315
|
| 1760 | 2024-06-20T21:03:15.865Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
file = illumos-utils/src/zone.rs:315
|
| 1761 | 2024-06-20T21:03:15.871Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
file = illumos-utils/src/zone.rs:354
|
| 1762 | 2024-06-20T21:03:15.876Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
file = illumos-utils/src/zone.rs:354
|
| 1763 | 2024-06-20T21:03:15.881Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
file = illumos-utils/src/zone.rs:354
|
| 1764 | 2024-06-20T21:03:15.888Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
file = illumos-utils/src/zone.rs:354
|
| 1765 | 2024-06-20T21:03:15.894Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
file = illumos-utils/src/zone.rs:354
|
| 1766 | 2024-06-20T21:03:15.899Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
file = illumos-utils/src/zone.rs:354
|
| 1767 | 2024-06-20T21:03:15.936Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:280
|
| 1768 | 2024-06-20T21:03:15.973Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:280
|
| 1769 | 2024-06-20T21:03:16.008Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:280
|
| 1770 | 2024-06-20T21:03:16.039Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:280
|
| 1771 | 2024-06-20T21:03:16.070Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:280
|
| 1772 | 2024-06-20T21:03:16.072Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:305
|
| 1773 | 2024-06-20T21:03:16.072Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:197
|
| 1774 | 2024-06-20T21:03:16.119Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:393
port = Port { inner: PortInner { name: "opte1", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 159, 92]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: "vopte1" } }
|
| 1775 | 2024-06-20T21:03:16.119Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = external_dns
|
| 1776 | 2024-06-20T21:03:16.121Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:305
|
| 1777 | 2024-06-20T21:03:16.121Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:197
|
| 1778 | 2024-06-20T21:03:16.163Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:393
port = Port { inner: PortInner { name: "opte2", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 200, 82]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: "vopte2" } }
|
| 1779 | 2024-06-20T21:03:16.163Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = external_dns
|
| 1780 | 2024-06-20T21:03:16.165Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:305
|
| 1781 | 2024-06-20T21:03:16.165Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:197
|
| 1782 | 2024-06-20T21:03:16.216Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:393
port = Port { inner: PortInner { name: "opte3", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 187, 155]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte3" } }
|
| 1783 | 2024-06-20T21:03:16.216Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = nexus
|
| 1784 | 2024-06-20T21:03:16.218Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:305
|
| 1785 | 2024-06-20T21:03:16.218Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:197
|
| 1786 | 2024-06-20T21:03:16.274Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:393
port = Port { inner: PortInner { name: "opte4", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 187, 37]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte4" } }
|
| 1787 | 2024-06-20T21:03:16.274Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = nexus
|
| 1788 | 2024-06-20T21:03:16.277Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:305
|
| 1789 | 2024-06-20T21:03:16.277Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:197
|
| 1790 | 2024-06-20T21:03:16.382Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:393
port = Port { inner: PortInner { name: "opte5", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 132, 182]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte5" } }
|
| 1791 | 2024-06-20T21:03:16.382Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = nexus
|
| 1792 | 2024-06-20T21:03:16.386Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 1793 | 2024-06-20T21:03:16.386Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 1794 | 2024-06-20T21:03:16.386Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 1795 | 2024-06-20T21:03:16.386Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 1796 | 2024-06-20T21:03:16.386Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 1797 | 2024-06-20T21:03:16.386Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 1798 | 2024-06-20T21:03:16.388Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 1799 | 2024-06-20T21:03:16.388Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 1800 | 2024-06-20T21:03:16.388Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 1801 | 2024-06-20T21:03:16.388Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 1802 | 2024-06-20T21:03:16.393Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 1803 | 2024-06-20T21:03:16.396Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1804 | 2024-06-20T21:03:16.439Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 1805 | 2024-06-20T21:03:16.446Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 1806 | 2024-06-20T21:03:16.450Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 1807 | 2024-06-20T21:03:16.453Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 1808 | 2024-06-20T21:03:16.454Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1809 | 2024-06-20T21:03:16.454Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1810 | 2024-06-20T21:03:16.454Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1811 | 2024-06-20T21:03:16.456Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1812 | 2024-06-20T21:03:16.692Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
file = illumos-utils/src/zone.rs:315
|
| 1813 | 2024-06-20T21:03:16.705Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
file = illumos-utils/src/zone.rs:315
|
| 1814 | 2024-06-20T21:03:16.718Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
file = illumos-utils/src/zone.rs:315
|
| 1815 | 2024-06-20T21:03:16.729Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
file = illumos-utils/src/zone.rs:315
|
| 1816 | 2024-06-20T21:03:16.738Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
file = illumos-utils/src/zone.rs:315
|
| 1817 | 2024-06-20T21:03:16.745Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
file = illumos-utils/src/zone.rs:354
|
| 1818 | 2024-06-20T21:03:16.752Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
file = illumos-utils/src/zone.rs:354
|
| 1819 | 2024-06-20T21:03:16.764Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
file = illumos-utils/src/zone.rs:354
|
| 1820 | 2024-06-20T21:03:16.790Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
file = illumos-utils/src/zone.rs:354
|
| 1821 | 2024-06-20T21:03:16.798Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
file = illumos-utils/src/zone.rs:354
|
| 1822 | 2024-06-20T21:03:17.441Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 189.73659ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:417
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 1823 | 2024-06-20T21:03:19.104Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService20"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::f"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::f"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1824 | 2024-06-20T21:03:19.104Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1825 | 2024-06-20T21:03:19.114Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService13"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::11"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::11"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1826 | 2024-06-20T21:03:19.114Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1827 | 2024-06-20T21:03:19.526Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService11"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::18"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_9ff415f8-ea2c-44b4-b9a7-2a7cf086871c/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::18"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="d6243c9c-28ba-456b-a01b-a2c07e4b0778"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1828 | 2024-06-20T21:03:19.526Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService18"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::12"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_cc5f6a04-2111-4941-923c-8867d737d5fc/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::12"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="5d183d45-87a1-4bfb-8129-9ad77db53ec6"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1829 | 2024-06-20T21:03:19.527Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1830 | 2024-06-20T21:03:19.536Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService14"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::16"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_63627367-eb03-4016-8c1d-73768441f612/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::16"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="1b8598f7-0e51-43fc-aec8-a8065cfec4bd"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1831 | 2024-06-20T21:03:19.536Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService15"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::17"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_2e39fd13-5176-4455-b633-7660296b7576/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::17"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="8bdabefb-e9fe-4953-a72f-f95684016e11"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1832 | 2024-06-20T21:03:19.536Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService16"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::13"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_662892f9-e563-4653-bedf-02b82422ec9a/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::13"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="179d6874-1642-4d20-ac15-5a2f81005e64"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1833 | 2024-06-20T21:03:19.537Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1834 | 2024-06-20T21:03:19.544Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService19"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::15"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_339b70a8-c294-4120-9bd3-76df19004c3b/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::15"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="85cd351a-8bb8-40ca-8123-307fe1d2ab32"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1835 | 2024-06-20T21:03:19.544Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService22"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::19"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_39d4ca15-26b5-4bd0-bbf6-fd95c2268c88/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::19"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="cbba5461-ef57-4038-80a8-1461874dafc1"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1836 | 2024-06-20T21:03:19.544Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService23"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::1a"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_ac756826-7d23-4a48-8f48-471e91447b90/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::1a"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="cbeb7ab9-351e-4d28-929e-a2967081db19"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1837 | 2024-06-20T21:03:19.544Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1838 | 2024-06-20T21:03:19.550Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1839 | 2024-06-20T21:03:19.556Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1840 | 2024-06-20T21:03:19.561Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1841 | 2024-06-20T21:03:19.567Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1842 | 2024-06-20T21:03:19.573Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1843 | 2024-06-20T21:03:19.853Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 25.545999ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1844 | 2024-06-20T21:03:19.864Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 37.863591ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1845 | 2024-06-20T21:03:19.874Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 41.21796ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1846 | 2024-06-20T21:03:19.883Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 27.056564ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1847 | 2024-06-20T21:03:19.892Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 66.91804ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1848 | 2024-06-20T21:03:19.901Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 145.331095ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1849 | 2024-06-20T21:03:19.911Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 53.1211ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1850 | 2024-06-20T21:03:19.982Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 48.314392ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1851 | 2024-06-20T21:03:20.962Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 73.261776ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1852 | 2024-06-20T21:03:21.022Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 58.251445ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1853 | 2024-06-20T21:03:21.118Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 60.583295ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1854 | 2024-06-20T21:03:21.140Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 67.337163ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1855 | 2024-06-20T21:03:21.181Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 106.397274ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1856 | 2024-06-20T21:03:21.225Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 79.609362ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1857 | 2024-06-20T21:03:21.262Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 80.32332ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1858 | 2024-06-20T21:03:21.305Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 192.523732ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1859 | 2024-06-20T21:03:21.354Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 137.738369ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1860 | 2024-06-20T21:03:21.389Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 121.581918ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1861 | 2024-06-20T21:03:21.422Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 135.97216ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1862 | 2024-06-20T21:03:21.447Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 61.809567ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1863 | 2024-06-20T21:03:21.467Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 116.345841ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1864 | 2024-06-20T21:03:21.481Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 246.375343ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1865 | 2024-06-20T21:03:21.494Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 157.649077ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1866 | 2024-06-20T21:03:21.506Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 150.959271ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1867 | 2024-06-20T21:03:21.519Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 270.440554ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1868 | 2024-06-20T21:03:21.532Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 320.382259ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1869 | 2024-06-20T21:03:21.544Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 161.139831ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1870 | 2024-06-20T21:03:21.556Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 141.861824ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1871 | 2024-06-20T21:03:21.567Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 207.214266ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1872 | 2024-06-20T21:03:21.578Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 290.1443ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1873 | 2024-06-20T21:03:21.595Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 216.2842ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1874 | 2024-06-20T21:03:21.663Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 447.491814ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1875 | 2024-06-20T21:03:21.673Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 456.671393ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1876 | 2024-06-20T21:03:21.711Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 577.805639ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1877 | 2024-06-20T21:03:21.722Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 291.544327ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1878 | 2024-06-20T21:03:21.738Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 491.397306ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1879 | 2024-06-20T21:03:21.785Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 539.046465ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1880 | 2024-06-20T21:03:21.801Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 333.85385ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1881 | 2024-06-20T21:03:21.824Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 374.693174ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1882 | 2024-06-20T21:03:21.864Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 1.082735017s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1883 | 2024-06-20T21:03:21.880Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 337.187973ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1884 | 2024-06-20T21:03:22.024Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 545.483344ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1885 | 2024-06-20T21:03:22.125Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 640.984417ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1886 | 2024-06-20T21:03:22.144Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 1.164358616s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1887 | 2024-06-20T21:03:22.156Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 1.177390426s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1888 | 2024-06-20T21:03:22.212Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 890.785238ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1889 | 2024-06-20T21:03:22.231Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 656.097919ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1890 | 2024-06-20T21:03:22.244Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 739.409942ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1891 | 2024-06-20T21:03:22.304Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 510.742753ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1892 | 2024-06-20T21:03:22.337Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 779.082459ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1893 | 2024-06-20T21:03:22.585Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 977.108397ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1894 | 2024-06-20T21:03:22.779Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 995.247401ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1895 | 2024-06-20T21:03:22.827Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 1.146931198s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1896 | 2024-06-20T21:03:22.899Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 612.852842ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1897 | 2024-06-20T21:03:22.959Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 917.184665ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1898 | 2024-06-20T21:03:22.997Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 508.106255ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1899 | 2024-06-20T21:03:23.122Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 903.47965ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1900 | 2024-06-20T21:03:23.265Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 571.049689ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1901 | 2024-06-20T21:03:23.353Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 1.224458076s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1902 | 2024-06-20T21:03:23.386Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 927.361498ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1903 | 2024-06-20T21:03:23.763Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 847.739341ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1904 | 2024-06-20T21:03:23.793Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 908.793681ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1905 | 2024-06-20T21:03:23.811Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 824.184098ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1906 | 2024-06-20T21:03:23.832Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 1.25055885s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1907 | 2024-06-20T21:03:23.845Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 1.138441651s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1908 | 2024-06-20T21:03:23.889Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 619.197384ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1909 | 2024-06-20T21:03:23.985Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 640.803462ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1910 | 2024-06-20T21:03:24.037Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 1.368708968s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1911 | 2024-06-20T21:03:24.419Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 1.161106517s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1912 | 2024-06-20T21:03:31.281Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 982.356959ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1913 | 2024-06-20T21:03:31.705Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 1.394200262s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1914 | 2024-06-20T21:03:31.715Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 1.015108198s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1915 | 2024-06-20T21:03:31.727Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 591.53401ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1916 | 2024-06-20T21:03:31.741Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 1.11630847s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1917 | 2024-06-20T21:03:31.753Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 980.191971ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1918 | 2024-06-20T21:03:31.766Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 791.558293ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1919 | 2024-06-20T21:03:31.779Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 1.318067651s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1920 | 2024-06-20T21:03:31.791Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 808.343473ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1921 | 2024-06-20T21:03:31.803Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 972.345652ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1922 | 2024-06-20T21:03:31.816Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 1.255596223s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1923 | 2024-06-20T21:03:32.333Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 1.004339548s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1924 | 2024-06-20T21:03:32.574Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 1.297954017s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1925 | 2024-06-20T21:03:32.614Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 897.459569ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1926 | 2024-06-20T21:03:32.659Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 543.714777ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:417
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 1927 | 2024-06-20T21:03:32.746Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 628.396062ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1928 | 2024-06-20T21:03:32.760Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 586.624847ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1929 | 2024-06-20T21:03:32.788Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 569.032058ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1930 | 2024-06-20T21:03:32.837Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 1.239100629s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1931 | 2024-06-20T21:03:32.870Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 702.898067ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1932 | 2024-06-20T21:03:33.087Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 654.715784ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1933 | 2024-06-20T21:03:33.110Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 656.336974ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1934 | 2024-06-20T21:03:33.349Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 1.191382849s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1935 | 2024-06-20T21:03:33.358Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 1.088130968s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1936 | 2024-06-20T21:03:33.367Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 1.277865312s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1937 | 2024-06-20T21:03:33.385Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 1.319256232s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1938 | 2024-06-20T21:03:33.522Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 1.485838953s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1939 | 2024-06-20T21:03:33.584Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 1.340783984s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1940 | 2024-06-20T21:03:33.752Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 1.357947476s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1941 | 2024-06-20T21:03:33.778Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 1.178399746s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1942 | 2024-06-20T21:03:33.885Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 1.477269291s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1943 | 2024-06-20T21:03:34.093Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 698.256274ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1944 | 2024-06-20T21:03:34.456Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 654.426425ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1945 | 2024-06-20T21:03:34.550Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 1.243740282s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1946 | 2024-06-20T21:03:34.655Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 832.200305ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1947 | 2024-06-20T21:03:34.713Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 756.844485ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1948 | 2024-06-20T21:03:34.800Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 1.229114712s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1949 | 2024-06-20T21:03:34.935Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 591.986899ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1950 | 2024-06-20T21:03:34.966Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 524.920683ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1951 | 2024-06-20T21:03:35.019Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 650.414073ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1952 | 2024-06-20T21:03:35.120Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 1.208416394s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1953 | 2024-06-20T21:03:35.128Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 1.488799204s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1954 | 2024-06-20T21:03:35.280Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService27"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::a"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte3"/>
<propval type="astring" name="gateway" value="172.30.2.1"/>
<propval type="astring" name="ip" value="172.30.2.5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1955 | 2024-06-20T21:03:35.280Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService28"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::b"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte4"/>
<propval type="astring" name="gateway" value="172.30.2.1"/>
<propval type="astring" name="ip" value="172.30.2.6"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1956 | 2024-06-20T21:03:35.301Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1957 | 2024-06-20T21:03:35.306Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1958 | 2024-06-20T21:03:35.373Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 737.092071ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1959 | 2024-06-20T21:03:35.462Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 47.880963ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1960 | 2024-06-20T21:03:35.480Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 577.331175ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1961 | 2024-06-20T21:03:35.487Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 30.41174ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1962 | 2024-06-20T21:03:35.495Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 752.941181ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1963 | 2024-06-20T21:03:35.501Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 1.273984677s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1964 | 2024-06-20T21:03:35.518Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 123.928288ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1965 | 2024-06-20T21:03:35.525Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 126.211916ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1966 | 2024-06-20T21:03:35.536Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 1.472053497s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1967 | 2024-06-20T21:03:35.653Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 279.500271ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1968 | 2024-06-20T21:03:35.662Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 231.983882ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1969 | 2024-06-20T21:03:35.678Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 1.489717488s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1970 | 2024-06-20T21:03:35.803Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 1.193664781s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1971 | 2024-06-20T21:03:35.903Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 345.981492ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1972 | 2024-06-20T21:03:35.941Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 329.058987ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1973 | 2024-06-20T21:03:36.039Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 731.074249ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1974 | 2024-06-20T21:03:36.068Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 635.394175ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1975 | 2024-06-20T21:03:36.119Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 627.026354ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1976 | 2024-06-20T21:03:36.258Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 636.287346ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1977 | 2024-06-20T21:03:36.267Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 992.192232ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1978 | 2024-06-20T21:03:36.279Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 747.05749ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1979 | 2024-06-20T21:03:36.338Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 762.159123ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1980 | 2024-06-20T21:03:36.627Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 1.371815148s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1981 | 2024-06-20T21:03:36.713Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 1.459866812s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 1982 | 2024-06-20T21:03:36.755Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 1.196006586s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1983 | 2024-06-20T21:03:36.778Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 1.293205339s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1984 | 2024-06-20T21:03:36.786Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 1.188790535s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1985 | 2024-06-20T21:03:36.904Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 723.805668ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1986 | 2024-06-20T21:03:37.006Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 1.092122096s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 1987 | 2024-06-20T21:03:37.018Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 1.038138858s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1988 | 2024-06-20T21:03:37.035Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 1.4221208s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 1989 | 2024-06-20T21:03:37.109Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 984.620237ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1990 | 2024-06-20T21:03:37.176Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 1.172378344s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 1991 | 2024-06-20T21:03:37.269Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 1.337577396s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 1992 | 2024-06-20T21:03:37.636Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 1.402415705s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 1993 | 2024-06-20T21:03:37.961Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 993.423281ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 1994 | 2024-06-20T21:03:37.983Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 607.926098ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 1995 | 2024-06-20T21:03:38.008Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 1.161000083s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 1996 | 2024-06-20T21:03:38.065Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 1.2925928s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 1997 | 2024-06-20T21:03:38.080Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 704.901808ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 1998 | 2024-06-20T21:03:38.103Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 844.197377ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 1999 | 2024-06-20T21:03:38.110Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 1.4589659s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2000 | 2024-06-20T21:03:38.183Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 1.426869803s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2001 | 2024-06-20T21:03:38.229Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService17"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::10"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::10"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2002 | 2024-06-20T21:03:38.230Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2003 | 2024-06-20T21:03:38.391Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 1.329114889s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2004 | 2024-06-20T21:03:38.523Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 724.861352ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2005 | 2024-06-20T21:03:38.614Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 1.411025863s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 2006 | 2024-06-20T21:03:38.634Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 1.046084175s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2007 | 2024-06-20T21:03:38.808Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 37.381091ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2008 | 2024-06-20T21:03:38.822Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 757.015255ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2009 | 2024-06-20T21:03:38.824Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService21"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::14"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_f32a5037-29da-4375-a8f1-8e9a8b5d3778/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::14"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="d30b7370-7ee8-47d9-bd15-3310cca1c8e3"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2010 | 2024-06-20T21:03:38.824Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2011 | 2024-06-20T21:03:38.858Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 56.841943ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2012 | 2024-06-20T21:03:38.925Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 278.536282ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2013 | 2024-06-20T21:03:38.959Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 1.037194827s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 2014 | 2024-06-20T21:03:38.969Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 951.604006ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2015 | 2024-06-20T21:03:38.988Z | INFO | SledAgent (ServiceManager): Profile for oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService10"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::d"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/oximeter">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="id" value="5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9"/>
<propval type="astring" name="address" value="[fd00:1122:3344:101::d]:12223"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2016 | 2024-06-20T21:03:38.988Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2017 | 2024-06-20T21:03:39.057Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 1.124414168s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 2018 | 2024-06-20T21:03:39.140Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 25.66976ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2019 | 2024-06-20T21:03:39.174Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 74.471255ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2020 | 2024-06-20T21:03:39.184Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 691.582176ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 2021 | 2024-06-20T21:03:39.197Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 68.234479ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2022 | 2024-06-20T21:03:39.215Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 285.277442ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2023 | 2024-06-20T21:03:39.258Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 1.383753356s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2024 | 2024-06-20T21:03:39.265Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 208.489721ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2025 | 2024-06-20T21:03:39.274Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 111.433304ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2026 | 2024-06-20T21:03:39.368Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 1.489684076s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 2027 | 2024-06-20T21:03:39.397Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 120.512721ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2028 | 2024-06-20T21:03:39.484Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 440.441955ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2029 | 2024-06-20T21:03:39.510Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 1.070845673s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2030 | 2024-06-20T21:03:39.527Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 264.081802ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2031 | 2024-06-20T21:03:39.578Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 759.662319ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2032 | 2024-06-20T21:03:39.589Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 934.168384ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2033 | 2024-06-20T21:03:39.619Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 1.230754548s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2034 | 2024-06-20T21:03:39.691Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 1.232953163s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2035 | 2024-06-20T21:03:39.729Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 1.192555905s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2036 | 2024-06-20T21:03:39.800Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 1.151250091s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2037 | 2024-06-20T21:03:39.885Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 661.541252ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 2038 | 2024-06-20T21:03:39.931Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 1.146620539s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2039 | 2024-06-20T21:03:39.939Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 928.122927ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2040 | 2024-06-20T21:03:40.006Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 742.196455ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 2041 | 2024-06-20T21:03:40.034Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 752.470388ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 2042 | 2024-06-20T21:03:40.193Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 1.183872535s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 2043 | 2024-06-20T21:03:40.227Z | INFO | SledAgent (ServiceManager): Profile for oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService25"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::8"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte1"/>
<propval type="astring" name="gateway" value="172.30.1.1"/>
<propval type="astring" name="ip" value="172.30.1.5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/external_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:101::8]:5353"/>
<propval type="astring" name="dns_address" value="172.30.1.5:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2044 | 2024-06-20T21:03:40.227Z | INFO | SledAgent (ServiceManager): Profile for oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService26"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::9"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte2"/>
<propval type="astring" name="gateway" value="172.30.1.1"/>
<propval type="astring" name="ip" value="172.30.1.6"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/external_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:101::9]:5353"/>
<propval type="astring" name="dns_address" value="172.30.1.6:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2045 | 2024-06-20T21:03:40.227Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2046 | 2024-06-20T21:03:40.232Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2047 | 2024-06-20T21:03:40.238Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService24"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::c"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte5"/>
<propval type="astring" name="gateway" value="172.30.2.1"/>
<propval type="astring" name="ip" value="172.30.2.7"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2048 | 2024-06-20T21:03:40.238Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2049 | 2024-06-20T21:03:40.352Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 1.381062342s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2050 | 2024-06-20T21:03:40.413Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 25.516666ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2051 | 2024-06-20T21:03:40.439Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 35.305985ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2052 | 2024-06-20T21:03:40.455Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 68.923778ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2053 | 2024-06-20T21:03:40.465Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 105.367297ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2054 | 2024-06-20T21:03:40.486Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 108.251514ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2055 | 2024-06-20T21:03:40.536Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 1.045087373s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2056 | 2024-06-20T21:03:40.548Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 63.885125ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2057 | 2024-06-20T21:03:40.563Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 1.306503108s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 2058 | 2024-06-20T21:03:40.584Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 225.64069ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2059 | 2024-06-20T21:03:40.596Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 1.189472998s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2060 | 2024-06-20T21:03:40.609Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 299.421332ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2061 | 2024-06-20T21:03:40.626Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 188.772977ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2062 | 2024-06-20T21:03:40.652Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 1.098500653s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2063 | 2024-06-20T21:03:40.759Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 780.126316ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 2064 | 2024-06-20T21:03:40.798Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 1.062098798s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 2065 | 2024-06-20T21:03:40.821Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 574.669268ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2066 | 2024-06-20T21:03:40.829Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 238.773964ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2067 | 2024-06-20T21:03:40.858Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 814.068793ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2068 | 2024-06-20T21:03:40.867Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 1.216641956s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 2069 | 2024-06-20T21:03:40.875Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 732.959634ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2070 | 2024-06-20T21:03:40.919Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 449.43068ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2071 | 2024-06-20T21:03:40.931Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 796.772851ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2072 | 2024-06-20T21:03:40.939Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 1.05604165s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2073 | 2024-06-20T21:03:40.961Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 1.324731355s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2074 | 2024-06-20T21:03:41.077Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 745.418817ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2075 | 2024-06-20T21:03:41.087Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 1.011111133s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2076 | 2024-06-20T21:03:41.454Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 463.953452ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2077 | 2024-06-20T21:03:41.477Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 827.902746ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2078 | 2024-06-20T21:03:41.504Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 1.439132434s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 2079 | 2024-06-20T21:03:41.551Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 1.223667493s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 2080 | 2024-06-20T21:03:41.591Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 791.560107ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2081 | 2024-06-20T21:03:41.618Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 831.854937ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2082 | 2024-06-20T21:03:41.684Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 781.177563ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2083 | 2024-06-20T21:03:41.738Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 1.201763344s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2084 | 2024-06-20T21:03:41.745Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 735.44573ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2085 | 2024-06-20T21:03:41.788Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 891.315141ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2086 | 2024-06-20T21:03:41.796Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 1.084291368s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2087 | 2024-06-20T21:03:41.830Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 1.451209908s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2088 | 2024-06-20T21:03:41.869Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 1.383076132s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 2089 | 2024-06-20T21:03:41.878Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 1.420960252s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 2090 | 2024-06-20T21:03:41.926Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 547.240143ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2091 | 2024-06-20T21:03:42.008Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 1.475448878s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2092 | 2024-06-20T21:03:42.093Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 591.804041ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 2093 | 2024-06-20T21:03:42.107Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 737.517282ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2094 | 2024-06-20T21:03:42.296Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 895.38242ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2095 | 2024-06-20T21:03:42.313Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 1.407310821s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2096 | 2024-06-20T21:03:42.393Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 1.134388361s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2097 | 2024-06-20T21:03:42.477Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 576.197494ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2098 | 2024-06-20T21:03:42.511Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 536.650928ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2099 | 2024-06-20T21:03:42.543Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 1.31811965s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2100 | 2024-06-20T21:03:42.656Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 724.299898ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2101 | 2024-06-20T21:03:42.688Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 700.88977ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2102 | 2024-06-20T21:03:42.716Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 786.975255ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 2103 | 2024-06-20T21:03:42.799Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 1.498905614s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 2104 | 2024-06-20T21:03:42.860Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 781.245344ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2105 | 2024-06-20T21:03:42.890Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 1.462244939s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2106 | 2024-06-20T21:03:42.948Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 963.148871ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2107 | 2024-06-20T21:03:42.956Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 1.301482423s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 2108 | 2024-06-20T21:03:43.058Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 1.335706179s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2109 | 2024-06-20T21:03:43.066Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 1.227186328s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2110 | 2024-06-20T21:03:43.201Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 956.316874ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2111 | 2024-06-20T21:03:43.262Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 1.294948695s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 2112 | 2024-06-20T21:03:43.291Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 1.035322083s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2113 | 2024-06-20T21:03:43.308Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 1.432030656s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 2114 | 2024-06-20T21:03:43.389Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 1.349299958s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2115 | 2024-06-20T21:03:43.398Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 762.895341ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2116 | 2024-06-20T21:03:43.493Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 655.297669ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2117 | 2024-06-20T21:03:43.512Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 529.574751ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 2118 | 2024-06-20T21:03:43.536Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 559.533107ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2119 | 2024-06-20T21:03:43.641Z | INFO | SledAgent (ServiceManager): Profile for oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService12"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::e"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/clickhouse">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::e"/>
<propval type="astring" name="listen_port" value="8123"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2120 | 2024-06-20T21:03:43.642Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2121 | 2024-06-20T21:03:43.655Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 1.167119122s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2122 | 2024-06-20T21:03:43.731Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 1.081721789s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2123 | 2024-06-20T21:03:43.865Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194") failed: Property not found. retry in 68.939476ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2124 | 2024-06-20T21:03:43.873Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 1.034787457s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2125 | 2024-06-20T21:03:43.921Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 791.811479ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2126 | 2024-06-20T21:03:43.943Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194") failed: Property not found. retry in 74.166269ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2127 | 2024-06-20T21:03:44.030Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194") failed: Property not found. retry in 236.563105ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2128 | 2024-06-20T21:03:44.054Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 1.375281577s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 2129 | 2024-06-20T21:03:44.106Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 1.132699772s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2130 | 2024-06-20T21:03:44.159Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 512.256771ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2131 | 2024-06-20T21:03:44.166Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 1.266955618s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2132 | 2024-06-20T21:03:44.174Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 776.275767ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2133 | 2024-06-20T21:03:44.267Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 807.649471ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 2134 | 2024-06-20T21:03:44.278Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194") failed: Property not found. retry in 215.916638ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2135 | 2024-06-20T21:03:44.302Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 665.146634ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2136 | 2024-06-20T21:03:44.310Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 1.066544542s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 2137 | 2024-06-20T21:03:44.335Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 929.413843ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2138 | 2024-06-20T21:03:44.361Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 513.132824ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2139 | 2024-06-20T21:03:44.403Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 1.248768235s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2140 | 2024-06-20T21:03:44.504Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194") failed: Property not found. retry in 1.197351726s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2141 | 2024-06-20T21:03:44.568Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 932.038832ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 2142 | 2024-06-20T21:03:44.682Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 1.195798647s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2143 | 2024-06-20T21:03:44.725Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 568.165016ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2144 | 2024-06-20T21:03:44.750Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 625.246555ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2145 | 2024-06-20T21:03:44.760Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 539.380624ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 2146 | 2024-06-20T21:03:44.826Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 1.276875877s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2147 | 2024-06-20T21:03:44.835Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 1.452910398s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2148 | 2024-06-20T21:03:44.886Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 933.050924ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2149 | 2024-06-20T21:03:44.921Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 596.350275ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2150 | 2024-06-20T21:03:44.962Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 1.3441941s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2151 | 2024-06-20T21:03:44.980Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 756.391846ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2152 | 2024-06-20T21:03:45.086Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 879.478906ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 2153 | 2024-06-20T21:03:45.249Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 1.313215786s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2154 | 2024-06-20T21:03:45.276Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 1.036084886s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2155 | 2024-06-20T21:03:45.306Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 1.267885531s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2156 | 2024-06-20T21:03:45.317Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 1.48916468s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 2157 | 2024-06-20T21:03:45.389Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 876.103866ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2158 | 2024-06-20T21:03:45.400Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 801.860662ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 2159 | 2024-06-20T21:03:45.441Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 1.247479915s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 2160 | 2024-06-20T21:03:45.450Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 1.455962876s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2161 | 2024-06-20T21:03:45.511Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 928.039599ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 2162 | 2024-06-20T21:03:45.529Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 568.838838ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2163 | 2024-06-20T21:03:45.665Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 768.152987ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2164 | 2024-06-20T21:03:45.714Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194") failed: Property not found. retry in 1.030752394s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2165 | 2024-06-20T21:03:45.746Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 1.299772176s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2166 | 2024-06-20T21:03:45.832Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 1.439261849s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2167 | 2024-06-20T21:03:45.890Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 1.427375538s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2168 | 2024-06-20T21:03:45.978Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 1.345379182s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 2169 | 2024-06-20T21:03:46.114Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 1.271107803s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2170 | 2024-06-20T21:03:46.127Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 731.059312ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2171 | 2024-06-20T21:03:46.218Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 571.106702ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 2172 | 2024-06-20T21:03:46.282Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 1.101159745s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2173 | 2024-06-20T21:03:46.306Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 607.968495ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2174 | 2024-06-20T21:03:46.322Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 1.305815861s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2175 | 2024-06-20T21:03:46.335Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 1.41247478s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2176 | 2024-06-20T21:03:46.446Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 864.398312ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2177 | 2024-06-20T21:03:46.458Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 1.113484696s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 2178 | 2024-06-20T21:03:46.575Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 896.891302ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2179 | 2024-06-20T21:03:46.588Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 880.354979ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2180 | 2024-06-20T21:03:46.702Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 1.288545588s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 2181 | 2024-06-20T21:03:46.763Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194") failed: Property not found. retry in 562.383001ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2182 | 2024-06-20T21:03:46.817Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 1.161134242s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 2183 | 2024-06-20T21:03:46.838Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 938.964703ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 2184 | 2024-06-20T21:03:46.873Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 573.547208ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2185 | 2024-06-20T21:03:46.922Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 1.39618237s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2186 | 2024-06-20T21:03:46.938Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 1.482236025s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2187 | 2024-06-20T21:03:47.063Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 655.537892ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2188 | 2024-06-20T21:03:47.286Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 959.506715ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2189 | 2024-06-20T21:03:47.326Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 1.361580157s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2190 | 2024-06-20T21:03:47.342Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194") failed: Property not found. retry in 972.313721ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2191 | 2024-06-20T21:03:47.359Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 688.451232ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2192 | 2024-06-20T21:03:47.398Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 896.888941ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 2193 | 2024-06-20T21:03:47.420Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 800.526628ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2194 | 2024-06-20T21:03:47.435Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 1.026274126s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2195 | 2024-06-20T21:03:47.462Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 1.254295214s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2196 | 2024-06-20T21:03:47.482Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 1.097545682s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2197 | 2024-06-20T21:03:47.496Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 565.426244ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2198 | 2024-06-20T21:03:47.591Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 1.463606746s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 2199 | 2024-06-20T21:03:47.643Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 785.466542ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2200 | 2024-06-20T21:03:47.734Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 1.394829671s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2201 | 2024-06-20T21:03:47.765Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 1.132634955s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2202 | 2024-06-20T21:03:47.792Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 1.324701106s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 2203 | 2024-06-20T21:03:47.993Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 608.872979ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 2204 | 2024-06-20T21:03:48.003Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 507.027864ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 2205 | 2024-06-20T21:03:48.060Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 510.01587ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2206 | 2024-06-20T21:03:48.074Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 1.434285672s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2207 | 2024-06-20T21:03:48.230Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 983.971513ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2208 | 2024-06-20T21:03:48.254Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 1.32324448s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2209 | 2024-06-20T21:03:48.303Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 590.740098ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 2210 | 2024-06-20T21:03:48.306Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 1.228244326s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:417
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2211 | 2024-06-20T21:03:48.325Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194") failed: Property not found. retry in 749.165552ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2212 | 2024-06-20T21:03:48.334Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 1.367027824s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2213 | 2024-06-20T21:03:48.431Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 618.726557ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2214 | 2024-06-20T21:03:48.439Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 1.363603471s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2215 | 2024-06-20T21:03:48.471Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 991.408772ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2216 | 2024-06-20T21:03:48.522Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 1.224537525s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 2217 | 2024-06-20T21:03:48.581Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 1.224818615s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2218 | 2024-06-20T21:03:48.592Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 595.781535ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2219 | 2024-06-20T21:03:48.613Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 941.137332ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 2220 | 2024-06-20T21:03:48.700Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 1.434374581s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2221 | 2024-06-20T21:03:48.731Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 1.319162301s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2222 | 2024-06-20T21:03:48.906Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 1.151372909s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 2223 | 2024-06-20T21:03:48.915Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 651.918445ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2224 | 2024-06-20T21:03:49.061Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 1.064637365s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2225 | 2024-06-20T21:03:49.070Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 534.039786ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 2226 | 2024-06-20T21:03:49.085Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194") failed: Property not found. retry in 1.079784424s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2227 | 2024-06-20T21:03:49.127Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 727.177707ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 2228 | 2024-06-20T21:03:49.139Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 1.133037892s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2229 | 2024-06-20T21:03:49.204Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 645.47389ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2230 | 2024-06-20T21:03:49.231Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 807.886057ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2231 | 2024-06-20T21:03:49.478Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 896.683176ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2232 | 2024-06-20T21:03:49.524Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 1.054172383s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2233 | 2024-06-20T21:03:49.579Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 1.339975024s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 2234 | 2024-06-20T21:03:49.592Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 506.132016ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2235 | 2024-06-20T21:03:49.605Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 521.511905ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2236 | 2024-06-20T21:03:49.619Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 1.340214625s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 2237 | 2024-06-20T21:03:49.717Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 748.502804ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2238 | 2024-06-20T21:03:49.760Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3") failed: Property not found. retry in 1.137419923s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 2239 | 2024-06-20T21:03:49.818Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 1.375925232s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2240 | 2024-06-20T21:03:49.830Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 666.163794ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2241 | 2024-06-20T21:03:49.864Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 1.333342569s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2242 | 2024-06-20T21:03:49.878Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 1.447675915s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 2243 | 2024-06-20T21:03:50.053Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 1.086845057s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2244 | 2024-06-20T21:03:50.068Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 1.047366782s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2245 | 2024-06-20T21:03:50.081Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 1.294005844s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 2246 | 2024-06-20T21:03:50.115Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 572.363945ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2247 | 2024-06-20T21:03:50.139Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 726.078029ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2248 | 2024-06-20T21:03:50.151Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 972.239467ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2249 | 2024-06-20T21:03:50.163Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 1.425307663s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2250 | 2024-06-20T21:03:50.181Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194") failed: Property not found. retry in 698.005444ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2251 | 2024-06-20T21:03:50.288Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 519.129111ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2252 | 2024-06-20T21:03:50.391Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 588.576008ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2253 | 2024-06-20T21:03:50.482Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 680.207344ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2254 | 2024-06-20T21:03:50.512Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 1.241518857s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2255 | 2024-06-20T21:03:50.594Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 1.440570726s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2256 | 2024-06-20T21:03:50.702Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 623.860472ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2257 | 2024-06-20T21:03:50.827Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 1.05191964s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2258 | 2024-06-20T21:03:50.886Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 1.168289979s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2259 | 2024-06-20T21:03:50.899Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194") failed: Property not found. retry in 907.048435ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2260 | 2024-06-20T21:03:50.933Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4") failed: Property not found. retry in 1.435896212s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 2261 | 2024-06-20T21:03:50.980Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5") failed: Property not found. retry in 1.086534966s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 2262 | 2024-06-20T21:03:50.998Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 1.159065855s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2263 | 2024-06-20T21:03:51.133Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 1.147001564s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2264 | 2024-06-20T21:03:51.149Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958") failed: Property not found. retry in 1.39398864s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2265 | 2024-06-20T21:03:51.163Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235") failed: Property not found. retry in 1.272041672s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2266 | 2024-06-20T21:03:51.176Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9") failed: Property not found. retry in 734.636472ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2267 | 2024-06-20T21:03:51.208Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 651.847576ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2268 | 2024-06-20T21:03:51.226Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385") failed: Property not found. retry in 1.144215799s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2269 | 2024-06-20T21:03:51.336Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 944.39492ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2270 | 2024-06-20T21:03:51.345Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 820.115446ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 2271 | 2024-06-20T21:03:51.390Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 579.635928ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 2272 | 2024-06-20T21:03:51.606Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df") failed: Property not found. retry in 756.365795ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2273 | 2024-06-20T21:03:51.771Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9") failed: Property not found. retry in 639.129353ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2274 | 2024-06-20T21:03:51.822Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194") failed: Property not found. retry in 1.249143196s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2275 | 2024-06-20T21:03:51.879Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d") failed: Property not found. retry in 710.094681ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2276 | 2024-06-20T21:03:51.898Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f") failed: Property not found. retry in 1.248462682s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2277 | 2024-06-20T21:03:51.989Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4") failed: Property not found. retry in 668.013059ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 2278 | 2024-06-20T21:03:52.054Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0") failed: Property not found. retry in 1.16487531s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2279 | 2024-06-20T21:03:52.074Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc") failed: Property not found. retry in 530.409835ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2280 | 2024-06-20T21:03:52.176Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec") failed: Property not found. retry in 839.43138ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2281 | 2024-06-20T21:03:52.198Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc") failed: Property not found. retry in 1.266760339s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 2282 | 2024-06-20T21:03:52.302Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57") failed: Property not found. retry in 1.114632587s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2283 | 2024-06-20T21:03:52.331Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217") failed: Property not found. retry in 760.146797ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2284 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_oximeter_5e3f1ed4-0f1b-4a1d-9959-8f40d4c97af9
|
| 2285 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_crucible_634cce40-bafd-4e0d-96ed-afdcb1d112b0
|
| 2286 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_external_dns_2d41aa6a-e613-43ba-b0d0-5cc8f685a958
|
| 2287 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_clickhouse_25b89476-359e-47f8-bea2-3de9e5e99194
|
| 2288 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_crucible_pantry_fe5b0415-63cc-4561-8e18-8b38bd5aa235
|
| 2289 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_crucible_2e9691b8-007b-4520-947b-a0537ec7b2ec
|
| 2290 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_external_dns_4e960e23-20ba-48e5-9697-e299814c1217
|
| 2291 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_crucible_646a904f-1208-4787-8b83-6ad524b58ce3
|
| 2292 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_crucible_2d29a557-7bee-4fe1-8b8e-8c4309441fe4
|
| 2293 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_crucible_pantry_70ea324e-1d3b-451f-92e2-3dd4893ba7df
|
| 2294 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_crucible_afaff3f3-797c-44e8-bbf7-7a0b7c0c5ffc
|
| 2295 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_crucible_b9bdba62-62b5-45f2-8e6d-7415e56d8efc
|
| 2296 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_crucible_pantry_271e109e-182e-4626-b37a-fed0542ab9a5
|
| 2297 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_nexus_ef10530a-0825-4479-8014-6a7d3078262d
|
| 2298 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_crucible_3b7765ae-cc70-4a27-9b47-752b8a12d69f
|
| 2299 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_nexus_4e4b5bd4-ec5c-4863-9626-ecd0b57498b9
|
| 2300 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_nexus_0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57
|
| 2301 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_crucible_bc18ef60-2412-4f2c-9cab-691f92a94385
|
| 2302 | 2024-06-20T21:03:53.582Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3056
zone = oxz_crucible_83db2fa4-dc9c-4ccd-a225-33f67333fdb4
|
| 2303 | 2024-06-20T21:03:53.596Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 2304 | 2024-06-20T21:03:53.608Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 2305 | 2024-06-20T21:03:53.627Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 38770561
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:63067
req_id = 89d83b6a-6603-4f04-998a-845270675675
response_code = 204
uri = /omicron-zones
|
| 2306 | 2024-06-20T21:03:53.642Z | INFO | SledAgent (RSS): Finished setting up services
file = sled-agent/src/rack_setup/service.rs:1244
|
| 2307 | 2024-06-20T21:03:53.643Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/.rss-plan-completed.marker.tmp
file = common/src/ledger.rs:199
|
| 2308 | 2024-06-20T21:03:53.644Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/.rss-plan-completed.marker.tmp
file = common/src/ledger.rs:199
|
| 2309 | 2024-06-20T21:03:53.647Z | INFO | SledAgent (RSS): Handing off control to Nexus
file = sled-agent/src/rack_setup/service.rs:681
|
| 2310 | 2024-06-20T21:03:53.656Z | INFO | SledAgent (RSS): Nexus address: [fd00:1122:3344:101::c]:12221
file = sled-agent/src/rack_setup/service.rs:694
|
| 2311 | 2024-06-20T21:03:53.708Z | INFO | SledAgent (RSS): rack_network_config: RackNetworkConfigV2 {
bfd: [],
bgp: [],
infra_ip_first: 10.151.2.174,
infra_ip_last: 10.151.2.174,
ports: [
PortConfigV2 {
addresses: [
UplinkAddressConfig {
address: V4(
Ipv4Net {
addr: 10.151.2.174,
width: 24,
},
),
vlan_id: None,
},
],
autoneg: false,
bgp_peers: [],
port: "qsfp0",
routes: [
RouteConfig {
destination: V4(
Ipv4Net {
addr: 0.0.0.0,
width: 0,
},
),
nexthop: 10.151.2.100,
vlan_id: None,
},
],
switch: Switch0,
uplink_port_fec: None,
uplink_port_speed: Speed40G,
},
],
rack_subnet: Ipv6Net {
addr: fd00:1122:3344:100::,
width: 56,
},
}
file = sled-agent/src/rack_setup/service.rs:825
|
| 2312 | 2024-06-20T21:03:54.015Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2313 | 2024-06-20T21:03:54.020Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 2.167037376s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = oximeter/producer/src/lib.rs:417
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2314 | 2024-06-20T21:03:54.074Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2315 | 2024-06-20T21:03:54.143Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2316 | 2024-06-20T21:03:54.240Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2317 | 2024-06-20T21:03:54.359Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2318 | 2024-06-20T21:03:54.503Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2319 | 2024-06-20T21:03:54.784Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2320 | 2024-06-20T21:03:55.120Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2321 | 2024-06-20T21:03:55.434Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2322 | 2024-06-20T21:03:55.740Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2323 | 2024-06-20T21:03:56.025Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2324 | 2024-06-20T21:03:56.221Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 3.241920943s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = oximeter/producer/src/lib.rs:417
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2325 | 2024-06-20T21:03:56.833Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2326 | 2024-06-20T21:03:57.817Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2327 | 2024-06-20T21:03:58.761Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2328 | 2024-06-20T21:03:59.494Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 11.518302996s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = oximeter/producer/src/lib.rs:417
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2329 | 2024-06-20T21:04:00.274Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2330 | 2024-06-20T21:04:01.962Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2331 | 2024-06-20T21:04:02.390Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:40172
|
| 2332 | 2024-06-20T21:04:02.393Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 2265
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:40172
req_id = 7a6f10bb-1dbb-4440-8aef-5a1c7ba5e2d5
response_code = 200
uri = /baseboard
|
| 2333 | 2024-06-20T21:04:03.464Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/0f15006d-98e4-48dc-8b6c-3e866ca1aace/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:887
|
| 2334 | 2024-06-20T21:04:06.276Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "878b509d-2856-408d-9955-9cf78990231a", "content-length": "173", "date": "Thu, 20 Jun 2024 21:04:05 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"ebe60f78-650f-482d-ab07-6cce79648e59\"", request_id: "878b509d-2856-408d-9955-9cf78990231a" }
file = sled-agent/src/rack_setup/service.rs:887
|
| 2335 | 2024-06-20T21:04:07.997Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "b6f035b1-3cb0-481b-a04c-506c346f6d35", "content-length": "173", "date": "Thu, 20 Jun 2024 21:04:07 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"ebe60f78-650f-482d-ab07-6cce79648e59\"", request_id: "b6f035b1-3cb0-481b-a04c-506c346f6d35" }
file = sled-agent/src/rack_setup/service.rs:887
|
| 2336 | 2024-06-20T21:04:11.000Z | INFO | SledAgent (NexusNotifierTask): Successfully put SledAgentInfo to nexus
file = sled-agent/src/nexus.rs:592
new_generation = 1
old_generation = 1
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2337 | 2024-06-20T21:04:11.120Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:60176
|
| 2338 | 2024-06-20T21:04:11.122Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2339 | 2024-06-20T21:04:11.208Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 87547
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:60176
req_id = c099309e-e61d-4d46-ae31-9acfa14d1f0f
response_code = 200
uri = /inventory
|
| 2340 | 2024-06-20T21:04:11.210Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2341 | 2024-06-20T21:04:11.210Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2342 | 2024-06-20T21:04:11.213Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2343 | 2024-06-20T21:04:11.214Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 6166
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:60176
req_id = 0d3476d3-4ebc-4afe-aaf3-00198f97ea1c
response_code = 200
uri = /omicron-zones
|
| 2344 | 2024-06-20T21:04:12.352Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:62775
|
| 2345 | 2024-06-20T21:04:12.360Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:524
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2346 | 2024-06-20T21:04:12.360Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2347 | 2024-06-20T21:04:12.360Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2348 | 2024-06-20T21:04:12.360Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2349 | 2024-06-20T21:04:12.360Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte0"
rules = []
|
| 2350 | 2024-06-20T21:04:12.360Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2351 | 2024-06-20T21:04:12.360Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2352 | 2024-06-20T21:04:12.360Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 8632
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:62775
req_id = 7634ceaf-07d5-4960-8aa8-d2994af2ca93
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2353 | 2024-06-20T21:04:12.363Z | INFO | SledAgent (RSS): Handoff to Nexus is complete
file = sled-agent/src/rack_setup/service.rs:897
|
| 2354 | 2024-06-20T21:04:12.563Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:56312
|
| 2355 | 2024-06-20T21:04:12.563Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2356 | 2024-06-20T21:04:12.653Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 90109
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:56312
req_id = f59214a1-52da-4842-bdec-65f7dfff397c
response_code = 200
uri = /inventory
|
| 2357 | 2024-06-20T21:04:12.653Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2358 | 2024-06-20T21:04:12.654Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2359 | 2024-06-20T21:04:12.654Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2360 | 2024-06-20T21:04:12.654Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 730
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:56312
req_id = 552763ed-9207-4197-b81e-dcb25b5414a9
response_code = 200
uri = /omicron-zones
|
| 2361 | 2024-06-20T21:04:14.100Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:54696
|
| 2362 | 2024-06-20T21:04:14.100Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:524
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2363 | 2024-06-20T21:04:14.100Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2364 | 2024-06-20T21:04:14.100Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2365 | 2024-06-20T21:04:14.100Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2366 | 2024-06-20T21:04:14.100Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte0"
rules = []
|
| 2367 | 2024-06-20T21:04:14.100Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2368 | 2024-06-20T21:04:14.100Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2369 | 2024-06-20T21:04:14.100Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 355
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:54696
req_id = bbb3cf32-f000-4abc-abdc-c79aa5960cc2
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2370 | 2024-06-20T21:04:14.100Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:60456
|
| 2371 | 2024-06-20T21:04:14.101Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:524
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2372 | 2024-06-20T21:04:14.101Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2373 | 2024-06-20T21:04:14.101Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2374 | 2024-06-20T21:04:14.101Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2375 | 2024-06-20T21:04:14.101Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte0"
rules = []
|
| 2376 | 2024-06-20T21:04:14.101Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2377 | 2024-06-20T21:04:14.101Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2378 | 2024-06-20T21:04:14.101Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 901
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:60456
req_id = fd3a7096-666b-4343-8efa-3628f317655a
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2379 | 2024-06-20T21:04:14.167Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:52179
|
| 2380 | 2024-06-20T21:04:14.167Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:524
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2381 | 2024-06-20T21:04:14.167Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2382 | 2024-06-20T21:04:14.167Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2383 | 2024-06-20T21:04:14.167Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2384 | 2024-06-20T21:04:14.167Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte0"
rules = []
|
| 2385 | 2024-06-20T21:04:14.167Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2386 | 2024-06-20T21:04:14.167Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2387 | 2024-06-20T21:04:14.167Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 326
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:52179
req_id = de25622d-99b3-4f52-ad30-dbc74407b368
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2388 | 2024-06-20T21:04:32.427Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:62807
|
| 2389 | 2024-06-20T21:04:32.428Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 633
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:62807
req_id = 11c91ebf-1fc5-4d9c-a3fd-a606c7c3c446
response_code = 200
uri = /baseboard
|
| 2390 | 2024-06-20T21:04:33.882Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:63762
|
| 2391 | 2024-06-20T21:04:33.883Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 1061
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:63762
req_id = dff00a56-b63e-4998-b0e4-66c91984e0a3
response_code = 200
uri = /v2p
|
| 2392 | 2024-06-20T21:04:33.955Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:37106
|
| 2393 | 2024-06-20T21:04:33.955Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 113
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:37106
req_id = 3cd5e0d0-5150-4195-ba88-097b7b543f13
response_code = 200
uri = /v2p
|
| 2394 | 2024-06-20T21:04:33.967Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:59379
|
| 2395 | 2024-06-20T21:04:33.967Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 66
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:59379
req_id = 3a831457-9b18-4be9-bf8a-a0bb0d323230
response_code = 200
uri = /v2p
|
| 2396 | 2024-06-20T21:04:33.988Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:43871
|
| 2397 | 2024-06-20T21:04:34.106Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 118106
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:43871
req_id = 63f8b66b-609c-47ba-a28a-d55883fab2e6
response_code = 204
uri = /switch-ports
|
| 2398 | 2024-06-20T21:04:34.107Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:58572
|
| 2399 | 2024-06-20T21:04:34.107Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:34791
|
| 2400 | 2024-06-20T21:04:34.111Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 4516
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:43871
req_id = cbc15e02-6dc7-4179-832d-39f9da01eb82
response_code = 200
uri = /network-bootstore-config
|
| 2401 | 2024-06-20T21:04:34.137Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 2, current_generation: 1
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
| 2402 | 2024-06-20T21:04:34.138Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 2403 | 2024-06-20T21:04:34.142Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 2404 | 2024-06-20T21:04:34.143Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 2
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-sock-i86pc
|
| 2405 | 2024-06-20T21:04:34.143Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 7841
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:43871
req_id = ae904a90-0092-4581-883b-e556268ed7da
response_code = 204
uri = /network-bootstore-config
|
| 2406 | 2024-06-20T21:04:34.170Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 63572
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:34791
req_id = bb252c12-5f8d-4398-9cfc-2519143bf9bb
response_code = 204
uri = /switch-ports
|
| 2407 | 2024-06-20T21:04:34.174Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 640
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:34791
req_id = 3e7807be-f4fa-40f2-ab83-ef12253a6120
response_code = 200
uri = /network-bootstore-config
|
| 2408 | 2024-06-20T21:04:34.217Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 110621
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:58572
req_id = 95205284-3c95-45bb-9a2b-1aa8c58a2bdc
response_code = 204
uri = /switch-ports
|
| 2409 | 2024-06-20T21:04:34.219Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 795
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:58572
req_id = e621e481-57da-424e-9335-b5c0482b663e
response_code = 200
uri = /network-bootstore-config
|
| 2410 | 2024-06-20T21:04:41.164Z | INFO | SledAgent (dropshot): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:57395
remote_addr = [fd00:1122:3344:101::d]:61486
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2411 | 2024-06-20T21:04:41.170Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 4900
local_addr = [fd00:1122:3344:101::1]:57395
method = GET
remote_addr = [fd00:1122:3344:101::d]:61486
req_id = 0858d9ff-6917-47af-ac7b-8f1c38b80d8b
response_code = 200
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
uri = /ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2412 | 2024-06-20T21:05:02.470Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45700
|
| 2413 | 2024-06-20T21:05:02.470Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 111
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45700
req_id = d8f74c4b-31cc-442f-8930-22c35daf6735
response_code = 200
uri = /baseboard
|
| 2414 | 2024-06-20T21:05:04.296Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:49023
|
| 2415 | 2024-06-20T21:05:04.297Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 1070
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:49023
req_id = 7ac5eac5-605d-4717-ad96-00ec47a1012c
response_code = 200
uri = /v2p
|
| 2416 | 2024-06-20T21:05:04.378Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:62285
|
| 2417 | 2024-06-20T21:05:04.424Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 46707
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:62285
req_id = 91513619-2ecb-4218-b70f-84804625dfeb
response_code = 204
uri = /switch-ports
|
| 2418 | 2024-06-20T21:05:04.425Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 83
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:62285
req_id = 6de812e5-1c58-445c-bcdd-e12a8a0f2aa1
response_code = 200
uri = /network-bootstore-config
|
| 2419 | 2024-06-20T21:05:04.445Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:64817
|
| 2420 | 2024-06-20T21:05:04.445Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 164
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:64817
req_id = 6537e843-57b2-4ca6-96a4-cf50263fc90d
response_code = 200
uri = /v2p
|
| 2421 | 2024-06-20T21:05:04.469Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:38176
|
| 2422 | 2024-06-20T21:05:04.469Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 85
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:38176
req_id = b72df7f2-60fb-43ce-8d13-fe84f8dd4e3f
response_code = 200
uri = /v2p
|
| 2423 | 2024-06-20T21:05:04.557Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:48038
|
| 2424 | 2024-06-20T21:05:04.605Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 47518
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:48038
req_id = d4cfd5cd-d8bf-44a1-ae8b-f3bef5175d7e
response_code = 204
uri = /switch-ports
|
| 2425 | 2024-06-20T21:05:04.605Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:42226
|
| 2426 | 2024-06-20T21:05:04.608Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 3434
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:48038
req_id = 6d1e490a-752b-4aa5-ac84-01be98e5ddba
response_code = 200
uri = /network-bootstore-config
|
| 2427 | 2024-06-20T21:05:04.650Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 44883
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:42226
req_id = 6aed0dd6-df32-4d3a-b21e-c34369d677b7
response_code = 204
uri = /switch-ports
|
| 2428 | 2024-06-20T21:05:04.650Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 120
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:42226
req_id = 4d728f31-36e3-490b-9a91-75289d49daa7
response_code = 200
uri = /network-bootstore-config
|
| 2429 | 2024-06-20T21:05:11.163Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 503
local_addr = [fd00:1122:3344:101::1]:57395
method = GET
remote_addr = [fd00:1122:3344:101::d]:61486
req_id = 348e7c04-fc40-4853-ae4d-fde2425b3993
response_code = 200
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
uri = /ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2430 | 2024-06-20T21:05:32.502Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:41703
|
| 2431 | 2024-06-20T21:05:32.502Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 83
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:41703
req_id = cec5085a-1bcc-4380-8533-ba2e0bd952aa
response_code = 200
uri = /baseboard
|
| 2432 | 2024-06-20T21:05:33.889Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:48311
|
| 2433 | 2024-06-20T21:05:33.889Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 205
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:48311
req_id = 40b38eff-4f62-4e8e-981b-d542ef7a3b58
response_code = 200
uri = /v2p
|
| 2434 | 2024-06-20T21:05:33.908Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:65286
|
| 2435 | 2024-06-20T21:05:33.908Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 89
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:65286
req_id = 36867e02-ec4d-475d-bdb3-a7a88ea24435
response_code = 200
uri = /v2p
|
| 2436 | 2024-06-20T21:05:34.000Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:61622
|
| 2437 | 2024-06-20T21:05:34.050Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 49856
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:61622
req_id = 4331e659-cae9-4ff4-8010-6136f0cabc46
response_code = 204
uri = /switch-ports
|
| 2438 | 2024-06-20T21:05:34.050Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 107
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:61622
req_id = dc04f60b-c081-4451-a9e7-13afffdee771
response_code = 200
uri = /network-bootstore-config
|
| 2439 | 2024-06-20T21:05:34.062Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:34623
|
| 2440 | 2024-06-20T21:05:34.062Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 187
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:34623
req_id = d3720d67-3227-408c-8ae4-57cc4b844454
response_code = 200
uri = /v2p
|
| 2441 | 2024-06-20T21:05:34.071Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:42685
|
| 2442 | 2024-06-20T21:05:34.118Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 47930
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:42685
req_id = 037ed51d-2b6f-4b3b-9493-ab03d0b6bfc9
response_code = 204
uri = /switch-ports
|
| 2443 | 2024-06-20T21:05:34.119Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 111
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:42685
req_id = 9036fe6e-8997-4062-ab12-30a852db07ed
response_code = 200
uri = /network-bootstore-config
|
| 2444 | 2024-06-20T21:05:34.152Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:37658
|
| 2445 | 2024-06-20T21:05:34.202Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 49617
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:37658
req_id = 9ea4bc42-c2d2-457f-b9ff-2df1f909c2ef
response_code = 204
uri = /switch-ports
|
| 2446 | 2024-06-20T21:05:34.202Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 96
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:37658
req_id = 19f0a140-2a66-4669-9fda-cd8218110a0d
response_code = 200
uri = /network-bootstore-config
|
| 2447 | 2024-06-20T21:05:41.163Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 607
local_addr = [fd00:1122:3344:101::1]:57395
method = GET
remote_addr = [fd00:1122:3344:101::d]:61486
req_id = b607f1f5-1f1e-4df3-aa29-be1122b62eee
response_code = 200
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
uri = /ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2448 | 2024-06-20T21:06:02.536Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:52809
|
| 2449 | 2024-06-20T21:06:02.536Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 164
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:52809
req_id = c3d28e36-5b04-4594-b3cf-c689c1f61705
response_code = 200
uri = /baseboard
|
| 2450 | 2024-06-20T21:06:03.945Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:42711
|
| 2451 | 2024-06-20T21:06:03.945Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 199
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:42711
req_id = e6870dfd-7403-471f-90d6-395d5b80d79c
response_code = 200
uri = /v2p
|
| 2452 | 2024-06-20T21:06:03.957Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:43928
|
| 2453 | 2024-06-20T21:06:03.957Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 189
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:43928
req_id = c47db1d3-fd48-48fa-929c-2f5b13d0d6ef
response_code = 200
uri = /v2p
|
| 2454 | 2024-06-20T21:06:04.167Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:43674
|
| 2455 | 2024-06-20T21:06:04.217Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 50287
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:43674
req_id = d02c4d1a-e087-4757-a5bb-caa823d21eba
response_code = 204
uri = /switch-ports
|
| 2456 | 2024-06-20T21:06:04.218Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:63725
|
| 2457 | 2024-06-20T21:06:04.218Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 126
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:43674
req_id = a97d9b98-2db6-45c2-b4e3-a194bf1977b4
response_code = 200
uri = /network-bootstore-config
|
| 2458 | 2024-06-20T21:06:04.218Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:58009
|
| 2459 | 2024-06-20T21:06:04.222Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 4307
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:58009
req_id = 9854d76b-cbb3-466c-877c-9debb771dcd9
response_code = 200
uri = /v2p
|
| 2460 | 2024-06-20T21:06:04.266Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 47857
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:63725
req_id = cd9a2995-fc2a-4af7-9cc5-96bc146bcdd3
response_code = 204
uri = /switch-ports
|
| 2461 | 2024-06-20T21:06:04.267Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 756
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:63725
req_id = ee20af60-b519-41e9-b198-2f32cf1c81b7
response_code = 200
uri = /network-bootstore-config
|
| 2462 | 2024-06-20T21:06:04.364Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:36012
|
| 2463 | 2024-06-20T21:06:04.406Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 42552
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:36012
req_id = f0181d25-680e-44d0-a950-bd58a741f390
response_code = 204
uri = /switch-ports
|
| 2464 | 2024-06-20T21:06:04.407Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 604
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:36012
req_id = 0b6ca993-584f-43ca-ae51-067a9d8c7696
response_code = 200
uri = /network-bootstore-config
|
| 2465 | 2024-06-20T21:06:11.162Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 589
local_addr = [fd00:1122:3344:101::1]:57395
method = GET
remote_addr = [fd00:1122:3344:101::d]:61486
req_id = 32242d47-6a36-4bbd-8a56-86de79811b1c
response_code = 200
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
uri = /ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2466 | 2024-06-20T21:06:32.569Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:63073
|
| 2467 | 2024-06-20T21:06:32.570Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 61
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:63073
req_id = ee80c3b4-6765-437b-b7cf-515fffc9b781
response_code = 200
uri = /baseboard
|
| 2468 | 2024-06-20T21:06:33.907Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:45574
|
| 2469 | 2024-06-20T21:06:33.907Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 177
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:45574
req_id = 2474dd4e-522d-4445-8ce0-43c8015ecf77
response_code = 200
uri = /v2p
|
| 2470 | 2024-06-20T21:06:33.918Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:37076
|
| 2471 | 2024-06-20T21:06:33.918Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 98
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:37076
req_id = cc1d11f1-14c3-48b5-bd82-6fb539b1be24
response_code = 200
uri = /v2p
|
| 2472 | 2024-06-20T21:06:33.970Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:49796
|
| 2473 | 2024-06-20T21:06:33.970Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 87
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:49796
req_id = 6d4f9cc4-ddbf-4d62-844c-bc9dd484472d
response_code = 200
uri = /v2p
|
| 2474 | 2024-06-20T21:06:33.977Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:56744
|
| 2475 | 2024-06-20T21:06:34.028Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 50472
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:56744
req_id = 5eb711d6-ba8f-4c78-89b7-b5cbc5a96c8b
response_code = 204
uri = /switch-ports
|
| 2476 | 2024-06-20T21:06:34.028Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:38753
|
| 2477 | 2024-06-20T21:06:34.072Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 43781
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:38753
req_id = 78d45a7d-8bdb-4c56-bb15-5eafda66c789
response_code = 204
uri = /switch-ports
|
| 2478 | 2024-06-20T21:06:34.072Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 102
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:56744
req_id = 3704808a-0246-43b1-94e1-e950d87153ca
response_code = 200
uri = /network-bootstore-config
|
| 2479 | 2024-06-20T21:06:34.073Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 763
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:38753
req_id = 6ee61d4e-2942-4fb0-ba7b-2fe3da7c8471
response_code = 200
uri = /network-bootstore-config
|
| 2480 | 2024-06-20T21:06:34.074Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:50703
|
| 2481 | 2024-06-20T21:06:34.119Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 44822
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:50703
req_id = 4c055015-34ce-4cf8-a84b-8daeb5f2c0df
response_code = 204
uri = /switch-ports
|
| 2482 | 2024-06-20T21:06:34.119Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 158
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:50703
req_id = 213826a4-fdc9-4e92-babc-6a2a54d46ba9
response_code = 200
uri = /network-bootstore-config
|
| 2483 | 2024-06-20T21:06:41.163Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 791
local_addr = [fd00:1122:3344:101::1]:57395
method = GET
remote_addr = [fd00:1122:3344:101::d]:61486
req_id = 3077825a-cf68-4036-8027-e0c73767c0ae
response_code = 200
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
uri = /ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2484 | 2024-06-20T21:06:44.035Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 2485 | 2024-06-20T21:07:02.601Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:35315
|
| 2486 | 2024-06-20T21:07:02.601Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 93
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:35315
req_id = 2c5dfadc-50d0-49ce-9536-8ba3b8c967a7
response_code = 200
uri = /baseboard
|
| 2487 | 2024-06-20T21:07:03.947Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:36314
|
| 2488 | 2024-06-20T21:07:03.947Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 240
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:36314
req_id = 5e8ff083-b341-4b18-a83d-2828c6bd0d43
response_code = 200
uri = /v2p
|
| 2489 | 2024-06-20T21:07:04.030Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:51075
|
| 2490 | 2024-06-20T21:07:04.030Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 90
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:51075
req_id = 2e6c4012-d88d-43b8-896c-f117ae512f87
response_code = 200
uri = /v2p
|
| 2491 | 2024-06-20T21:07:04.153Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:65008
|
| 2492 | 2024-06-20T21:07:04.154Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 141
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:65008
req_id = 76b10f1a-c9f3-4e56-813c-e18e5a929199
response_code = 200
uri = /v2p
|
| 2493 | 2024-06-20T21:07:04.169Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:64277
|
| 2494 | 2024-06-20T21:07:04.218Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 48188
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:64277
req_id = 8f3fc463-f58a-4c0c-862b-b2a05c2a786c
response_code = 204
uri = /switch-ports
|
| 2495 | 2024-06-20T21:07:04.218Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 83
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:64277
req_id = d85be350-1b40-48f0-a735-57fcfac945db
response_code = 200
uri = /network-bootstore-config
|
| 2496 | 2024-06-20T21:07:04.348Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:57857
|
| 2497 | 2024-06-20T21:07:04.394Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 46042
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:57857
req_id = 767b2e81-51ac-4596-9573-82a656f8f241
response_code = 204
uri = /switch-ports
|
| 2498 | 2024-06-20T21:07:04.395Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:58539
|
| 2499 | 2024-06-20T21:07:04.398Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 3706
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:57857
req_id = ea5f1c89-ed92-4b48-986d-d629d3b0b396
response_code = 200
uri = /network-bootstore-config
|
| 2500 | 2024-06-20T21:07:04.442Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 46914
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:58539
req_id = 12fc1b2c-51ac-4597-a9f9-1b623aed02c8
response_code = 204
uri = /switch-ports
|
| 2501 | 2024-06-20T21:07:04.442Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 143
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:58539
req_id = 1c35a1df-88c8-4e58-9c48-30adc64c86c0
response_code = 200
uri = /network-bootstore-config
|
| 2502 | 2024-06-20T21:07:11.313Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 932
local_addr = [fd00:1122:3344:101::1]:57395
method = GET
remote_addr = [fd00:1122:3344:101::d]:61486
req_id = cef5e126-d809-4ca7-a0e6-43fcd461c861
response_code = 200
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
uri = /ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2503 | 2024-06-20T21:07:32.634Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:42572
|
| 2504 | 2024-06-20T21:07:32.635Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 56
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:42572
req_id = 8c51db21-fc44-4341-9aa5-540459d3db5c
response_code = 200
uri = /baseboard
|
| 2505 | 2024-06-20T21:07:33.902Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:48111
|
| 2506 | 2024-06-20T21:07:33.902Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 189
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:48111
req_id = 3558a8aa-eae0-4dd8-b3d3-f88f03a788b0
response_code = 200
uri = /v2p
|
| 2507 | 2024-06-20T21:07:33.921Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:40056
|
| 2508 | 2024-06-20T21:07:33.922Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 763
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:40056
req_id = 26a84ddd-871d-410e-bc4d-a69a6f73833d
response_code = 200
uri = /v2p
|
| 2509 | 2024-06-20T21:07:33.936Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:47225
|
| 2510 | 2024-06-20T21:07:33.936Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 92
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:47225
req_id = e6884968-fdb8-434b-af54-38f92642ed4e
response_code = 200
uri = /v2p
|
| 2511 | 2024-06-20T21:07:33.974Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:55635
|
| 2512 | 2024-06-20T21:07:34.022Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 48138
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:55635
req_id = be91d5d4-5a62-417f-89c2-c2a370067a26
response_code = 204
uri = /switch-ports
|
| 2513 | 2024-06-20T21:07:34.022Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:40397
|
| 2514 | 2024-06-20T21:07:34.022Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 128
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:55635
req_id = c176a5a6-2b66-4722-af64-843bc79fec6a
response_code = 200
uri = /network-bootstore-config
|
| 2515 | 2024-06-20T21:07:34.071Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 48443
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:40397
req_id = dcd42676-6bad-4e6f-b3bf-0ace6ec3134b
response_code = 204
uri = /switch-ports
|
| 2516 | 2024-06-20T21:07:34.071Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 121
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:40397
req_id = 4befdbcb-c043-44c9-8f1c-c78310051e0e
response_code = 200
uri = /network-bootstore-config
|
| 2517 | 2024-06-20T21:07:34.079Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:65135
|
| 2518 | 2024-06-20T21:07:34.126Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 47253
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:65135
req_id = aa5cc36a-f0ea-4650-8a6d-a90d429002a8
response_code = 204
uri = /switch-ports
|
| 2519 | 2024-06-20T21:07:34.127Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 117
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:65135
req_id = 218efa30-2a48-449f-bb7e-21b68f551b92
response_code = 200
uri = /network-bootstore-config
|
| 2520 | 2024-06-20T21:07:41.313Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 770
local_addr = [fd00:1122:3344:101::1]:57395
method = GET
remote_addr = [fd00:1122:3344:101::d]:61486
req_id = 3cef6e8f-1dd4-45d3-a7ba-1904302562d0
response_code = 200
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
uri = /ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2521 | 2024-06-20T21:08:02.667Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:62125
|
| 2522 | 2024-06-20T21:08:02.667Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 89
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:62125
req_id = 98a572d8-7402-4ede-b681-7a92ce45cc22
response_code = 200
uri = /baseboard
|
| 2523 | 2024-06-20T21:08:04.039Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:65478
|
| 2524 | 2024-06-20T21:08:04.040Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 224
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:65478
req_id = ddb756d5-c1c4-43c9-85e4-18224daef329
response_code = 200
uri = /v2p
|
| 2525 | 2024-06-20T21:08:04.084Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:34403
|
| 2526 | 2024-06-20T21:08:04.085Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 813
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:34403
req_id = 37a50fe0-dc43-494a-8750-fa46db4d8c8a
response_code = 200
uri = /v2p
|
| 2527 | 2024-06-20T21:08:04.085Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:53111
|
| 2528 | 2024-06-20T21:08:04.085Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 74
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:53111
req_id = c58761d4-efcb-4a92-84e1-04ba2019d9d1
response_code = 200
uri = /v2p
|
| 2529 | 2024-06-20T21:08:04.202Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:51434
|
| 2530 | 2024-06-20T21:08:04.248Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 46358
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:51434
req_id = 4eb555a9-6103-4756-80bd-4ac39c16a3a2
response_code = 204
uri = /switch-ports
|
| 2531 | 2024-06-20T21:08:04.249Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:37506
|
| 2532 | 2024-06-20T21:08:04.252Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 2850
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:51434
req_id = d2e05b5b-672d-4010-bc61-a556f1e6b3c5
response_code = 200
uri = /network-bootstore-config
|
| 2533 | 2024-06-20T21:08:04.291Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 42311
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:37506
req_id = 7a8dd1b2-3aef-48af-861e-79042b1dae11
response_code = 204
uri = /switch-ports
|
| 2534 | 2024-06-20T21:08:04.291Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 115
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:37506
req_id = e968a8eb-8572-4ec3-8484-b7fbea1915f0
response_code = 200
uri = /network-bootstore-config
|
| 2535 | 2024-06-20T21:08:04.361Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:33892
|
| 2536 | 2024-06-20T21:08:04.401Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 40029
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:33892
req_id = 5ac0e51d-c7ad-4cd7-b052-ba395938e658
response_code = 204
uri = /switch-ports
|
| 2537 | 2024-06-20T21:08:04.401Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 104
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:33892
req_id = 6eac2c57-1a11-4d72-bfee-7abe5071f677
response_code = 200
uri = /network-bootstore-config
|
| 2538 | 2024-06-20T21:08:11.313Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 678
local_addr = [fd00:1122:3344:101::1]:57395
method = GET
remote_addr = [fd00:1122:3344:101::d]:61486
req_id = 6c4388e2-5d44-400f-8265-460d024ec781
response_code = 200
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
uri = /ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2539 | 2024-06-20T21:08:32.701Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:64605
|
| 2540 | 2024-06-20T21:08:32.701Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 95
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:64605
req_id = d9b47a9f-1c96-455c-8f10-b2ee8c367b9d
response_code = 200
uri = /baseboard
|
| 2541 | 2024-06-20T21:08:33.900Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:53199
|
| 2542 | 2024-06-20T21:08:33.901Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 756
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:53199
req_id = 3fa0fb09-58fc-4f42-8204-a552ef90c255
response_code = 200
uri = /v2p
|
| 2543 | 2024-06-20T21:08:33.939Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:56261
|
| 2544 | 2024-06-20T21:08:33.940Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 868
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:56261
req_id = 295d942f-f349-4dc4-93b7-b4d026e02f64
response_code = 200
uri = /v2p
|
| 2545 | 2024-06-20T21:08:33.959Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:55116
|
| 2546 | 2024-06-20T21:08:33.959Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 82
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:55116
req_id = f67630ec-bb9d-4112-8db4-f2c2723f3be2
response_code = 200
uri = /v2p
|
| 2547 | 2024-06-20T21:08:34.055Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:60592
|
| 2548 | 2024-06-20T21:08:34.102Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 46525
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:60592
req_id = 6533a964-3b3e-4bd8-8251-7661e05cdb3f
response_code = 204
uri = /switch-ports
|
| 2549 | 2024-06-20T21:08:34.102Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:37570
|
| 2550 | 2024-06-20T21:08:34.102Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:33352
|
| 2551 | 2024-06-20T21:08:34.106Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 3330
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:60592
req_id = 520bdd4e-f194-4f20-8faa-4b3712555716
response_code = 200
uri = /network-bootstore-config
|
| 2552 | 2024-06-20T21:08:34.148Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 45232
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:33352
req_id = 601578b4-f8d2-45a0-ba3c-acb3cce88ce9
response_code = 204
uri = /switch-ports
|
| 2553 | 2024-06-20T21:08:34.152Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 966
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:33352
req_id = 2c9ce408-949f-49c9-b41d-3c3872aa7406
response_code = 200
uri = /network-bootstore-config
|
| 2554 | 2024-06-20T21:08:34.192Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 89103
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:37570
req_id = 89427da5-4cfb-44b2-8ffd-1ebfbce76977
response_code = 204
uri = /switch-ports
|
| 2555 | 2024-06-20T21:08:34.192Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 146
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:37570
req_id = fe61f620-b9fc-45e4-b13c-1b754c458051
response_code = 200
uri = /network-bootstore-config
|
| 2556 | 2024-06-20T21:08:41.313Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 819
local_addr = [fd00:1122:3344:101::1]:57395
method = GET
remote_addr = [fd00:1122:3344:101::d]:61486
req_id = dd9482ff-04c6-472e-a115-d351751b3544
response_code = 200
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
uri = /ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2557 | 2024-06-20T21:09:02.734Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:44784
|
| 2558 | 2024-06-20T21:09:02.734Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 74
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:44784
req_id = 2fa4cc9b-1cfb-4cfc-8cd3-52767c98cde3
response_code = 200
uri = /baseboard
|
| 2559 | 2024-06-20T21:09:04.059Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:56333
|
| 2560 | 2024-06-20T21:09:04.059Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 205
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:56333
req_id = 7ae40e25-81b5-4a96-b90d-5038b835bf7e
response_code = 200
uri = /v2p
|
| 2561 | 2024-06-20T21:09:04.067Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:54388
|
| 2562 | 2024-06-20T21:09:04.067Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 99
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:54388
req_id = 83ea037e-8473-4752-93f3-234ecfea6ec0
response_code = 200
uri = /v2p
|
| 2563 | 2024-06-20T21:09:04.081Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:45568
|
| 2564 | 2024-06-20T21:09:04.081Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:524
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2565 | 2024-06-20T21:09:04.081Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2566 | 2024-06-20T21:09:04.081Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2567 | 2024-06-20T21:09:04.081Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2568 | 2024-06-20T21:09:04.082Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte0"
rules = []
|
| 2569 | 2024-06-20T21:09:04.082Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2570 | 2024-06-20T21:09:04.082Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2571 | 2024-06-20T21:09:04.082Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 307
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:45568
req_id = 2da7b6a9-4949-45a7-87e8-c61a68a2e770
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2572 | 2024-06-20T21:09:04.105Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:60963
|
| 2573 | 2024-06-20T21:09:04.105Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 84
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:60963
req_id = 76df2d00-8a20-470f-a2ca-69587473b5c6
response_code = 200
uri = /v2p
|
| 2574 | 2024-06-20T21:09:04.122Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:58079
|
| 2575 | 2024-06-20T21:09:04.122Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:524
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2576 | 2024-06-20T21:09:04.122Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2577 | 2024-06-20T21:09:04.122Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2578 | 2024-06-20T21:09:04.122Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2579 | 2024-06-20T21:09:04.122Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte0"
rules = []
|
| 2580 | 2024-06-20T21:09:04.122Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2581 | 2024-06-20T21:09:04.122Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2582 | 2024-06-20T21:09:04.122Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 276
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:58079
req_id = 0f318854-72d1-4111-a28a-938431148524
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2583 | 2024-06-20T21:09:04.153Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:50837
|
| 2584 | 2024-06-20T21:09:04.153Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:524
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4f2d23fb-a78e-476f-9439-9d9b9b9ed1af, kind: Service { id: 4e960e23-20ba-48e5-9697-e299814c1217 }, name: Name("external-dns-4e960e23-20ba-48e5-9697-e299814c1217"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 82])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 7f03ccff-2d5f-49fb-8f37-625d5981e079, kind: Service { id: 2d41aa6a-e613-43ba-b0d0-5cc8f685a958 }, name: Name("external-dns-2d41aa6a-e613-43ba-b0d0-5cc8f685a958"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 92])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 412cf5a0-691c-4cd3-8f46-ffcc45d6f286, kind: Service { id: 4e4b5bd4-ec5c-4863-9626-ecd0b57498b9 }, name: Name("nexus-4e4b5bd4-ec5c-4863-9626-ecd0b57498b9"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 37])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d772999e-aa6b-4592-a99d-a41b433b650b, kind: Service { id: ef10530a-0825-4479-8014-6a7d3078262d }, name: Name("nexus-ef10530a-0825-4479-8014-6a7d3078262d"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ddfc9ab8-d17b-4a4c-bbc4-a819365fe8ea, kind: Service { id: 0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57 }, name: Name("nexus-0b8be9bf-e25e-49ba-bb5c-c0fbf3780d57"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 182])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2585 | 2024-06-20T21:09:04.153Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2586 | 2024-06-20T21:09:04.153Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2587 | 2024-06-20T21:09:04.153Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2588 | 2024-06-20T21:09:04.153Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte0"
rules = []
|
| 2589 | 2024-06-20T21:09:04.153Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2590 | 2024-06-20T21:09:04.153Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:543
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2591 | 2024-06-20T21:09:04.153Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 180
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:50837
req_id = 2a8a5e53-e54b-4909-b936-777c01d8057b
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2592 | 2024-06-20T21:09:04.185Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:61776
|
| 2593 | 2024-06-20T21:09:04.230Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 45023
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:61776
req_id = f6f640a2-4b45-4406-828e-906186cf8cc3
response_code = 204
uri = /switch-ports
|
| 2594 | 2024-06-20T21:09:04.230Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 98
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:61776
req_id = 79ad8b5c-cd2c-4cab-a68f-02c44a0b46a0
response_code = 200
uri = /network-bootstore-config
|
| 2595 | 2024-06-20T21:09:04.290Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:53928
|
| 2596 | 2024-06-20T21:09:04.333Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 43011
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:53928
req_id = 865f633c-7fb6-4ca1-b790-cdb6dd2c19d9
response_code = 204
uri = /switch-ports
|
| 2597 | 2024-06-20T21:09:04.333Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 114
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:53928
req_id = f06fa4a7-4a45-493d-b10c-c1e408a7353c
response_code = 200
uri = /network-bootstore-config
|
| 2598 | 2024-06-20T21:09:04.377Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:63234
|
| 2599 | 2024-06-20T21:09:04.423Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 45706
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:63234
req_id = 30bc5cc0-a7b5-489b-9e55-232be15dbf7f
response_code = 204
uri = /switch-ports
|
| 2600 | 2024-06-20T21:09:04.423Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 134
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:63234
req_id = f7673143-127b-4fcf-92db-6181d5b090f8
response_code = 200
uri = /network-bootstore-config
|
| 2601 | 2024-06-20T21:09:11.313Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 804
local_addr = [fd00:1122:3344:101::1]:57395
method = GET
remote_addr = [fd00:1122:3344:101::d]:61486
req_id = 3de133f2-ad25-4169-9d1c-2163b239641f
response_code = 200
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
uri = /ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2602 | 2024-06-20T21:09:32.767Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:35896
|
| 2603 | 2024-06-20T21:09:32.767Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 98
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:35896
req_id = 144d6697-6a7b-4144-836f-b031b849bf07
response_code = 200
uri = /baseboard
|
| 2604 | 2024-06-20T21:09:33.843Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:49843
|
| 2605 | 2024-06-20T21:09:33.843Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 206
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:49843
req_id = c3cc6cac-b941-4af6-8f1e-489f98e06e02
response_code = 200
uri = /v2p
|
| 2606 | 2024-06-20T21:09:33.887Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:38427
|
| 2607 | 2024-06-20T21:09:33.888Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 98
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:38427
req_id = 508f5efc-fdb0-4aa1-85de-791babb447e3
response_code = 200
uri = /v2p
|
| 2608 | 2024-06-20T21:09:33.917Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:54770
|
| 2609 | 2024-06-20T21:09:33.917Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 87
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:54770
req_id = 627cb264-dfda-4106-95e5-5c098ff2dfa6
response_code = 200
uri = /v2p
|
| 2610 | 2024-06-20T21:09:33.987Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:36563
|
| 2611 | 2024-06-20T21:09:34.035Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 48276
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:36563
req_id = 98524ffe-569a-49fb-a1f4-2a2cebe3d34a
response_code = 204
uri = /switch-ports
|
| 2612 | 2024-06-20T21:09:34.036Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:60140
|
| 2613 | 2024-06-20T21:09:34.039Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 3064
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:36563
req_id = b38d8967-3fc4-4b11-a785-2426eb5990f0
response_code = 200
uri = /network-bootstore-config
|
| 2614 | 2024-06-20T21:09:34.078Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:46467
|
| 2615 | 2024-06-20T21:09:34.081Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 45043
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:60140
req_id = 5fe266c0-647a-40ef-8603-d8b21b7c3ed3
response_code = 204
uri = /switch-ports
|
| 2616 | 2024-06-20T21:09:34.084Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 591
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:60140
req_id = 2132ae51-41e3-484d-a19d-d578ca4b8416
response_code = 200
uri = /network-bootstore-config
|
| 2617 | 2024-06-20T21:09:34.125Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 47082
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:46467
req_id = ca3e0d50-aa15-466f-8a2a-e05c5b8af2ac
response_code = 204
uri = /switch-ports
|
| 2618 | 2024-06-20T21:09:34.126Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 740
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:46467
req_id = 68e3b7b6-6509-45c0-a1b1-b1fe2d0eae7e
response_code = 200
uri = /network-bootstore-config
|
| 2619 | 2024-06-20T21:09:41.393Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 576
local_addr = [fd00:1122:3344:101::1]:57395
method = GET
remote_addr = [fd00:1122:3344:101::d]:61486
req_id = 64ffbcaa-a765-4ce0-a5b7-db89204e2d60
response_code = 200
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
uri = /ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2620 | 2024-06-20T21:10:02.799Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:38243
|
| 2621 | 2024-06-20T21:10:02.800Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 896
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:38243
req_id = 375160bc-ad13-4518-84a2-adda6a135c8f
response_code = 200
uri = /baseboard
|
| 2622 | 2024-06-20T21:10:04.002Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:50064
|
| 2623 | 2024-06-20T21:10:04.002Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 193
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:50064
req_id = a45f5abe-2e74-46f7-8b68-6a77a1f62809
response_code = 200
uri = /v2p
|
| 2624 | 2024-06-20T21:10:04.037Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:43843
|
| 2625 | 2024-06-20T21:10:04.037Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 82
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:43843
req_id = 4e06795a-5662-459a-82e5-07e439057b99
response_code = 200
uri = /v2p
|
| 2626 | 2024-06-20T21:10:04.136Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:51636
|
| 2627 | 2024-06-20T21:10:04.137Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 945
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:51636
req_id = 07ea9fed-1cec-4e5f-8912-919d62e43f00
response_code = 200
uri = /v2p
|
| 2628 | 2024-06-20T21:10:04.174Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:60842
|
| 2629 | 2024-06-20T21:10:04.220Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 45261
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:60842
req_id = d74cd2f8-4458-4bbf-b3d0-0e2619aa4381
response_code = 204
uri = /switch-ports
|
| 2630 | 2024-06-20T21:10:04.220Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:48694
|
| 2631 | 2024-06-20T21:10:04.223Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 3010
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:60842
req_id = 7c99f69f-62f8-4413-81bf-4a9702286fa0
response_code = 200
uri = /network-bootstore-config
|
| 2632 | 2024-06-20T21:10:04.265Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 45424
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:48694
req_id = 7c4ecb62-0576-416b-8083-9861001689eb
response_code = 204
uri = /switch-ports
|
| 2633 | 2024-06-20T21:10:04.266Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 719
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:48694
req_id = 0efe8610-5945-41b0-89bf-d35a36aba070
response_code = 200
uri = /network-bootstore-config
|
| 2634 | 2024-06-20T21:10:04.272Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:53121
|
| 2635 | 2024-06-20T21:10:04.312Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 40621
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:53121
req_id = e139cd81-6b85-43ad-880a-6b405429297f
response_code = 204
uri = /switch-ports
|
| 2636 | 2024-06-20T21:10:04.313Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 116
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:53121
req_id = f071d4be-8212-4962-a839-9c2fd8abbde7
response_code = 200
uri = /network-bootstore-config
|
| 2637 | 2024-06-20T21:10:11.393Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 764
local_addr = [fd00:1122:3344:101::1]:57395
method = GET
remote_addr = [fd00:1122:3344:101::d]:61486
req_id = d0872f7a-c982-4a09-8281-74ecdfe3792b
response_code = 200
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
uri = /ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2638 | 2024-06-20T21:10:26.302Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:49756
|
| 2639 | 2024-06-20T21:10:26.303Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 183
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:49756
req_id = 1820788d-dccc-4ffa-8878-5dd1fe3c6f13
response_code = 200
uri = /v2p
|
| 2640 | 2024-06-20T21:10:26.305Z | INFO | SledAgent (PortManager): Mapping virtual NIC to physical host
file = illumos-utils/src/opte/port_manager.rs:631
mapping = VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 253, 161, 31])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(364767) }
|
| 2641 | 2024-06-20T21:10:26.305Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 2159
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:49756
req_id = d4248476-d343-4b88-89c2-59efaf46d8d6
response_code = 204
uri = /v2p
|
| 2642 | 2024-06-20T21:10:26.481Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:59413
|
| 2643 | 2024-06-20T21:10:26.499Z | INFO | SledAgent (InstanceManager): ensuring instance is registered
file = sled-agent/src/instance_manager.rs:548
hardware = InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname("localshark") }, nics: [NetworkInterface { id: 00f7b15c-aa0f-4a2c-abdc-4eb0f625e0e6, kind: Instance { id: 5485f2f1-8413-45ff-96bf-5be990261b69 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 161, 31])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(364767), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 00f7b15c-aa0f-4a2c-abdc-4eb0f625e0e6, kind: Instance { id: 5485f2f1-8413-45ff-96bf-5be990261b69 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 161, 31])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(364767), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 00f7b15c-aa0f-4a2c-abdc-4eb0f625e0e6, kind: Instance { id: 5485f2f1-8413-45ff-96bf-5be990261b69 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 161, 31])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(364767), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(364767))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 00f7b15c-aa0f-4a2c-abdc-4eb0f625e0e6, kind: Instance { id: 5485f2f1-8413-45ff-96bf-5be990261b69 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 161, 31])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(364767), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: "nvme", name: "disk-d7ae972938b", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 361df455-f18c-4d8d-9e64-cf6f334e7d8c, read_only_parent: Some(Volume { block_size: 512, id: 52822563-b086-4841-8772-2d2abbcb75fc, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 45f47d0b-f9a3-40d4-802a-bbae50817674, key: Some("WEZ1GfhEF+yP/Z+QLfgAGEAnFNMXq+l0d3fBpxDAncI="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::12]:19002", "[fd00:1122:3344:101::1a]:19001", "[fd00:1122:3344:101::14]:19002"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 361df455-f18c-4d8d-9e64-cf6f334e7d8c, key: Some("QcvWQisnVwRcrXxEWuQt5A2w2frjGkx2ca9FDWY0zaA="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::14]:19003", "[fd00:1122:3344:101::16]:19000", "[fd00:1122:3344:101::1a]:19002"] } }] } }], cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAsTanUWNRYAABNqdRYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAALE2p1FjUWAAATanUWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI1NDg1ZjJmMS04NDEzLTQ1ZmYtOTZiZi01YmU5OTAyNjFiNjkiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJR0FHZlN2WE5XRnlaRWh0Nndmd0RpdDdITlN3eWNBNFV5aEFLeUkxUEx6cSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") }
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
instance_runtime = InstanceRuntimeState { propolis_id: Some(25502922-1e61-417e-93d4-52e06d0f813f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:10:26.071430140Z }
metadata = InstanceMetadata { silo_id: e7e2a46b-e8c0-4912-8dec-96727dd9e600, project_id: 555e75e0-b99a-4a7e-9e77-2bae94fdb809 }
propolis_addr = [fd00:1122:3344:101::1:0]:12400
propolis_id = 25502922-1e61-417e-93d4-52e06d0f813f
vmm_runtime = VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-06-20T21:10:26.009667Z }
|
| 2644 | 2024-06-20T21:10:26.499Z | INFO | SledAgent (InstanceManager): registering new instance
file = sled-agent/src/instance_manager.rs:582
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69 (instance)
|
| 2645 | 2024-06-20T21:10:26.502Z | INFO | SledAgent (InstanceManager): initializing new Instance
file = sled-agent/src/instance.rs:942
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
propolis_id = 25502922-1e61-417e-93d4-52e06d0f813f
state = InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname("localshark") }, nics: [NetworkInterface { id: 00f7b15c-aa0f-4a2c-abdc-4eb0f625e0e6, kind: Instance { id: 5485f2f1-8413-45ff-96bf-5be990261b69 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 161, 31])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(364767), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 00f7b15c-aa0f-4a2c-abdc-4eb0f625e0e6, kind: Instance { id: 5485f2f1-8413-45ff-96bf-5be990261b69 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 161, 31])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(364767), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 00f7b15c-aa0f-4a2c-abdc-4eb0f625e0e6, kind: Instance { id: 5485f2f1-8413-45ff-96bf-5be990261b69 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 161, 31])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(364767), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(364767))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 00f7b15c-aa0f-4a2c-abdc-4eb0f625e0e6, kind: Instance { id: 5485f2f1-8413-45ff-96bf-5be990261b69 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 161, 31])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(364767), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: "nvme", name: "disk-d7ae972938b", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 361df455-f18c-4d8d-9e64-cf6f334e7d8c, read_only_parent: Some(Volume { block_size: 512, id: 52822563-b086-4841-8772-2d2abbcb75fc, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 45f47d0b-f9a3-40d4-802a-bbae50817674, key: Some("WEZ1GfhEF+yP/Z+QLfgAGEAnFNMXq+l0d3fBpxDAncI="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::12]:19002", "[fd00:1122:3344:101::1a]:19001", "[fd00:1122:3344:101::14]:19002"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 361df455-f18c-4d8d-9e64-cf6f334e7d8c, key: Some("QcvWQisnVwRcrXxEWuQt5A2w2frjGkx2ca9FDWY0zaA="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::14]:19003", "[fd00:1122:3344:101::16]:19000", "[fd00:1122:3344:101::1a]:19002"] } }] } }], cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAsTanUWNRYAABNqdRYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAALE2p1FjUWAAATanUWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI1NDg1ZjJmMS04NDEzLTQ1ZmYtOTZiZi01YmU5OTAyNjFiNjkiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJR0FHZlN2WE5XRnlaRWh0Nndmd0RpdDdITlN3eWNBNFV5aEFLeUkxUEx6cSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(25502922-1e61-417e-93d4-52e06d0f813f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:10:26.071430140Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-06-20T21:10:26.009667Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }
|
| 2646 | 2024-06-20T21:10:26.503Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 21310
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:59413
req_id = 7a0076ce-bae8-4acf-aa2e-032c8c18d371
response_code = 200
uri = /instances/5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2647 | 2024-06-20T21:10:26.628Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:56650
|
| 2648 | 2024-06-20T21:10:26.630Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 1749
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:56650
req_id = cad79b87-82a9-4cf6-a073-35d35ed1f881
response_code = 200
uri = /instances/5485f2f1-8413-45ff-96bf-5be990261b69/state
|
| 2649 | 2024-06-20T21:10:26.654Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:393
port = Port { inner: PortInner { name: "opte6", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 253, 161, 31]), slot: 0, vni: Vni { inner: 364767 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: "vopte6" } }
|
| 2650 | 2024-06-20T21:10:26.654Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2651 | 2024-06-20T21:10:26.689Z | INFO | SledAgent (InstanceManager): Configuring new Omicron zone: oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
file = illumos-utils/src/zone.rs:315
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2652 | 2024-06-20T21:10:26.710Z | INFO | SledAgent (InstanceManager): Installing Omicron zone: oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
file = illumos-utils/src/zone.rs:354
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2653 | 2024-06-20T21:10:27.910Z | INFO | SledAgent (InstanceManager): Profile for oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="system/illumos/propolis-server">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlInstance0"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::1:0"/>
<propval type="astring" name="listen_port" value="12400"/>
<propval type="astring" name="metric_addr" value="[fd00:1122:3344:101::a]:12221"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2654 | 2024-06-20T21:10:27.911Z | INFO | SledAgent (InstanceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
zone = oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
|
| 2655 | 2024-06-20T21:10:28.112Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f") failed: Property not found. retry in 60.002873ms
file = illumos-utils/src/svc.rs:36
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
zone = oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
|
| 2656 | 2024-06-20T21:10:28.191Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f") failed: Property not found. retry in 135.959966ms
file = illumos-utils/src/svc.rs:36
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
zone = oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
|
| 2657 | 2024-06-20T21:10:28.349Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f") failed: Property not found. retry in 124.486705ms
file = illumos-utils/src/svc.rs:36
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
zone = oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
|
| 2658 | 2024-06-20T21:10:28.488Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f") failed: Property not found. retry in 278.674294ms
file = illumos-utils/src/svc.rs:36
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
zone = oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
|
| 2659 | 2024-06-20T21:10:28.780Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f") failed: Property not found. retry in 411.170396ms
file = illumos-utils/src/svc.rs:36
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
zone = oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
|
| 2660 | 2024-06-20T21:10:29.202Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f") failed: Property not found. retry in 593.960536ms
file = illumos-utils/src/svc.rs:36
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
zone = oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
|
| 2661 | 2024-06-20T21:10:29.806Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f") failed: Property not found. retry in 1.290610111s
file = illumos-utils/src/svc.rs:36
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
zone = oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
|
| 2662 | 2024-06-20T21:10:31.107Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f") failed: Property not found. retry in 1.393248169s
file = illumos-utils/src/svc.rs:36
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
zone = oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
|
| 2663 | 2024-06-20T21:10:32.508Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f") failed: Property not found. retry in 1.02779548s
file = illumos-utils/src/svc.rs:36
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
zone = oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
|
| 2664 | 2024-06-20T21:10:32.832Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:48295
|
| 2665 | 2024-06-20T21:10:32.832Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 69
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:48295
req_id = 566fe8f1-ce8e-4cfb-8151-4c7362d200cc
response_code = 200
uri = /baseboard
|
| 2666 | 2024-06-20T21:10:33.562Z | INFO | SledAgent (InstanceManager): Started propolis in zone: oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
file = sled-agent/src/instance.rs:1427
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2667 | 2024-06-20T21:10:33.570Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f") failed: Property not found. retry in 54.266619ms
file = illumos-utils/src/svc.rs:36
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2668 | 2024-06-20T21:10:33.635Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f") failed: Property not found. retry in 88.658353ms
file = illumos-utils/src/svc.rs:36
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2669 | 2024-06-20T21:10:33.735Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f") failed: Property not found. retry in 100.747468ms
file = illumos-utils/src/svc.rs:36
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2670 | 2024-06-20T21:10:33.847Z | INFO | SledAgent (InstanceManager): Propolis SMF service is online
file = sled-agent/src/instance.rs:1436
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2671 | 2024-06-20T21:10:33.881Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:53913
|
| 2672 | 2024-06-20T21:10:33.883Z | INFO | SledAgent (InstanceManager): Propolis HTTP server online
file = sled-agent/src/instance.rs:1450
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2673 | 2024-06-20T21:10:33.886Z | INFO | SledAgent (InstanceManager): Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAsTanUWNRYAABNqdRYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAALE2p1FjUWAAATanUWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI1NDg1ZjJmMS04NDEzLTQ1ZmYtOTZiZi01YmU5OTAyNjFiNjkiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJR0FHZlN2WE5XRnlaRWh0Nndmd0RpdDdITlN3eWNBNFV5aEFLeUkxUEx6cSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"), disks: [DiskRequest { device: "nvme", name: "disk-d7ae972938b", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 361df455-f18c-4d8d-9e64-cf6f334e7d8c, read_only_parent: Some(Volume { block_size: 512, id: 52822563-b086-4841-8772-2d2abbcb75fc, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 45f47d0b-f9a3-40d4-802a-bbae50817674, key: Some("WEZ1GfhEF+yP/Z+QLfgAGEAnFNMXq+l0d3fBpxDAncI="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::12]:19002", "[fd00:1122:3344:101::1a]:19001", "[fd00:1122:3344:101::14]:19002"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 361df455-f18c-4d8d-9e64-cf6f334e7d8c, key: Some("QcvWQisnVwRcrXxEWuQt5A2w2frjGkx2ca9FDWY0zaA="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::14]:19003", "[fd00:1122:3344:101::16]:19000", "[fd00:1122:3344:101::1a]:19002"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: "vopte6", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: "Test description", id: 5485f2f1-8413-45ff-96bf-5be990261b69, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 555e75e0-b99a-4a7e-9e77-2bae94fdb809, silo_id: e7e2a46b-e8c0-4912-8dec-96727dd9e600 }, name: "localshark", vcpus: 2 } }
file = sled-agent/src/instance.rs:689
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2674 | 2024-06-20T21:10:33.887Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:37886
|
| 2675 | 2024-06-20T21:10:33.889Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 2520
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:37886
req_id = e6b296e5-2b9e-447e-87f0-c56147116903
response_code = 200
uri = /v2p
|
| 2676 | 2024-06-20T21:10:33.925Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:33647
|
| 2677 | 2024-06-20T21:10:33.936Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:41550
|
| 2678 | 2024-06-20T21:10:33.936Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 114
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:41550
req_id = e60f1426-1487-4586-aaf6-932d2220b31c
response_code = 200
uri = /v2p
|
| 2679 | 2024-06-20T21:10:33.966Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:52660
|
| 2680 | 2024-06-20T21:10:33.999Z | INFO | SledAgent (InstanceManager): result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })
file = sled-agent/src/instance.rs:691
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2681 | 2024-06-20T21:10:33.999Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 118032
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:53913
req_id = fb458058-a0cb-46f3-a2c7-b1c297c7b738
response_code = 200
uri = /instances/5485f2f1-8413-45ff-96bf-5be990261b69/state
|
| 2682 | 2024-06-20T21:10:33.999Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 33594
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:52660
req_id = 47265ab8-2b1c-402a-bc12-2dcff479644e
response_code = 200
uri = /instances/5485f2f1-8413-45ff-96bf-5be990261b69/state
|
| 2683 | 2024-06-20T21:10:33.999Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 74734
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:33647
req_id = 700b4659-68c2-4208-a952-3a1070f1ec4e
response_code = 200
uri = /instances/5485f2f1-8413-45ff-96bf-5be990261b69/state
|
| 2684 | 2024-06-20T21:10:34.001Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-06-20T21:10:34.001371698Z }
file = sled-agent/src/instance.rs:573
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2685 | 2024-06-20T21:10:34.001Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:595
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
new_instance_state = InstanceRuntimeState { propolis_id: Some(25502922-1e61-417e-93d4-52e06d0f813f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:10:26.071430140Z }
new_vmm_state = VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-06-20T21:10:34.001371698Z }
propolis_id = 25502922-1e61-417e-93d4-52e06d0f813f
|
| 2686 | 2024-06-20T21:10:34.001Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:492
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(25502922-1e61-417e-93d4-52e06d0f813f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:10:26.071430140Z }, propolis_id: 25502922-1e61-417e-93d4-52e06d0f813f (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-06-20T21:10:34.001371698Z }, migration_state: None }
|
| 2687 | 2024-06-20T21:10:34.013Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:61535
|
| 2688 | 2024-06-20T21:10:34.013Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 112
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:61535
req_id = 37321b06-fa32-4b67-9ccf-93bd3d09fec2
response_code = 200
uri = /v2p
|
| 2689 | 2024-06-20T21:10:34.034Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:40078
|
| 2690 | 2024-06-20T21:10:34.085Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 50616
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:40078
req_id = 42815442-52d7-47e3-80ca-4996db53fa02
response_code = 204
uri = /switch-ports
|
| 2691 | 2024-06-20T21:10:34.085Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 151
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:40078
req_id = 9220fa12-f2f8-47d4-9e06-3d6a78722d05
response_code = 200
uri = /network-bootstore-config
|
| 2692 | 2024-06-20T21:10:34.086Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-06-20T21:10:34.086764010Z }
file = sled-agent/src/instance.rs:573
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2693 | 2024-06-20T21:10:34.086Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:595
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
new_instance_state = InstanceRuntimeState { propolis_id: Some(25502922-1e61-417e-93d4-52e06d0f813f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:10:26.071430140Z }
new_vmm_state = VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-06-20T21:10:34.086764010Z }
propolis_id = 25502922-1e61-417e-93d4-52e06d0f813f
|
| 2694 | 2024-06-20T21:10:34.086Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:492
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(25502922-1e61-417e-93d4-52e06d0f813f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:10:26.071430140Z }, propolis_id: 25502922-1e61-417e-93d4-52e06d0f813f (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-06-20T21:10:34.086764010Z }, migration_state: None }
|
| 2695 | 2024-06-20T21:10:34.103Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:34186
|
| 2696 | 2024-06-20T21:10:34.148Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 44246
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:34186
req_id = fd6bb251-0b6a-4078-bda2-326828cecd07
response_code = 204
uri = /switch-ports
|
| 2697 | 2024-06-20T21:10:34.148Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:34369
|
| 2698 | 2024-06-20T21:10:34.152Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 3431
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:34186
req_id = d8f5f4db-4c1c-4989-8a83-58bc12b9c8c2
response_code = 200
uri = /network-bootstore-config
|
| 2699 | 2024-06-20T21:10:34.192Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 43927
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:34369
req_id = 34205984-642a-46b7-b5b4-414e1c72b4bb
response_code = 204
uri = /switch-ports
|
| 2700 | 2024-06-20T21:10:34.192Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 126
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:34369
req_id = 8b996e11-3c6d-412f-aac1-6907dcc31b3b
response_code = 200
uri = /network-bootstore-config
|
| 2701 | 2024-06-20T21:10:41.392Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 469
local_addr = [fd00:1122:3344:101::1]:57395
method = GET
remote_addr = [fd00:1122:3344:101::d]:61486
req_id = acc4fcb7-689a-4596-a14e-854bf2b1f5a2
response_code = 200
sled_id = ebe60f78-650f-482d-ab07-6cce79648e59
uri = /ebe60f78-650f-482d-ab07-6cce79648e59
|
| 2702 | 2024-06-20T21:10:44.031Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:59341
|
| 2703 | 2024-06-20T21:10:44.031Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 641
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:59341
req_id = c42bce72-789e-427b-9fab-8a8dad503ab8
response_code = 200
uri = /instances/5485f2f1-8413-45ff-96bf-5be990261b69/state
|
| 2704 | 2024-06-20T21:10:44.032Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-06-20T21:10:44.032011964Z }
file = sled-agent/src/instance.rs:573
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2705 | 2024-06-20T21:10:44.032Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:595
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
new_instance_state = InstanceRuntimeState { propolis_id: Some(25502922-1e61-417e-93d4-52e06d0f813f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:10:26.071430140Z }
new_vmm_state = VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-06-20T21:10:44.032011964Z }
propolis_id = 25502922-1e61-417e-93d4-52e06d0f813f
|
| 2706 | 2024-06-20T21:10:44.032Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:492
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(25502922-1e61-417e-93d4-52e06d0f813f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:10:26.071430140Z }, propolis_id: 25502922-1e61-417e-93d4-52e06d0f813f (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-06-20T21:10:44.032011964Z }, migration_state: None }
|
| 2707 | 2024-06-20T21:10:44.086Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-06-20T21:10:44.086216872Z }
file = sled-agent/src/instance.rs:573
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2708 | 2024-06-20T21:10:44.086Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:595
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
new_instance_state = InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-20T21:10:44.086216872Z }
new_vmm_state = VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-06-20T21:10:44.086216872Z }
propolis_id = 25502922-1e61-417e-93d4-52e06d0f813f
|
| 2709 | 2024-06-20T21:10:44.086Z | INFO | SledAgent (InstanceManager): terminating VMM that has exited
file = sled-agent/src/instance.rs:610
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2710 | 2024-06-20T21:10:44.086Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2711 | 2024-06-20T21:10:44.087Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2712 | 2024-06-20T21:10:44.087Z | INFO | SledAgent (ZoneBundler): creating zone bundle
context = ZoneBundleContext { storage_dirs: ["/pool/int/7a99c490-997c-4274-94ca-e12a3ae10436/debug/bundle/zone", "/pool/int/9d01641f-20d1-417f-950c-b1e4c58d23b5/debug/bundle/zone"], cause: TerminatedInstance, extra_log_dirs: ["/pool/ext/cc5f6a04-2111-4941-923c-8867d737d5fc/crypt/debug", "/pool/ext/662892f9-e563-4653-bedf-02b82422ec9a/crypt/debug", "/pool/ext/f32a5037-29da-4375-a8f1-8e9a8b5d3778/crypt/debug", "/pool/ext/339b70a8-c294-4120-9bd3-76df19004c3b/crypt/debug", "/pool/ext/63627367-eb03-4016-8c1d-73768441f612/crypt/debug", "/pool/ext/2e39fd13-5176-4455-b633-7660296b7576/crypt/debug", "/pool/ext/9ff415f8-ea2c-44b4-b9a7-2a7cf086871c/crypt/debug", "/pool/ext/39d4ca15-26b5-4bd0-bbf6-fd95c2268c88/crypt/debug", "/pool/ext/ac756826-7d23-4a48-8f48-471e91447b90/crypt/debug"] }
file = sled-agent/src/zone_bundle.rs:437
zone_name = oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
|
| 2713 | 2024-06-20T21:10:44.716Z | INFO | SledAgent (ZoneBundler): finished zone bundle
file = sled-agent/src/zone_bundle.rs:1230
metadata = ZoneBundleMetadata { id: ZoneBundleId { zone_name: "oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f", bundle_id: 048bf1aa-645c-46fc-80f5-7cf69ec8359e }, time_created: 2024-06-20T21:10:44.088369679Z, version: 0, cause: TerminatedInstance }
|
| 2714 | 2024-06-20T21:10:44.716Z | WARN | SledAgent (InstanceManager): Halting and removing zone: oxz_propolis-server_25502922-1e61-417e-93d4-52e06d0f813f
file = sled-agent/src/instance.rs:784
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2715 | 2024-06-20T21:10:46.454Z | INFO | SledAgent (InstanceManager): halt_and_remove_logged: Previous zone state: Running
file = illumos-utils/src/zone.rs:269
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2716 | 2024-06-20T21:10:46.480Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:492
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-20T21:10:44.086216872Z }, propolis_id: 25502922-1e61-417e-93d4-52e06d0f813f (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-06-20T21:10:46.480290341Z }, migration_state: None }
|
| 2717 | 2024-06-20T21:10:46.631Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:65317
|
| 2718 | 2024-06-20T21:10:46.631Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 233
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:65317
req_id = b350e10e-5d80-49b8-95f9-5c6e0ec7cc7c
response_code = 200
uri = /v2p
|
| 2719 | 2024-06-20T21:10:46.857Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:492
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-20T21:10:44.086216872Z }, propolis_id: 25502922-1e61-417e-93d4-52e06d0f813f (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-06-20T21:10:46.480290341Z }, migration_state: None }
|
| 2720 | 2024-06-20T21:10:46.861Z | INFO | SledAgent (InstanceManager): State monitoring task complete
file = sled-agent/src/instance.rs:727
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2721 | 2024-06-20T21:10:46.898Z | ERRO | SledAgent (InstanceManager): Failed to publish state to Nexus, will not retry: Notification(Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "496951d2-493b-4edb-a68c-05995c5a3a5a", "content-length": "177", "date": "Thu, 20 Jun 2024 21:10:46 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: instance with id \"5485f2f1-8413-45ff-96bf-5be990261b69\"", request_id: "496951d2-493b-4edb-a68c-05995c5a3a5a" })
file = sled-agent/src/instance.rs:559
instance_id = 5485f2f1-8413-45ff-96bf-5be990261b69
|
| 2722 | 2024-06-20T21:10:46.993Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:59166
|
| 2723 | 2024-06-20T21:10:46.993Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878
latency_us = 166
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:59166
req_id = 9e36f410-56f1-4729-8ab5-869728baab9a
response_code = 200
uri = /v2p
|