| 1 | | [ May 29 21:46:43 Enabled. ] |
| 2 | | [ May 29 21:46:43 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] |
| 3 | | [ May 29 21:46:43 Method "start" exited with status 0. ] |
| 4 | | [ May 29 21:46:43 Rereading configuration. ] |
| 5 | | [ May 29 21:46:43 No 'refresh' method defined. Treating as :true. ] |
| 6 | | note: configured to log to "/dev/stdout" |
| 7 | 2024-05-29T21:46:43.499Z | INFO | SledAgent: Ensuring contract reaper thread
file = illumos-utils/src/running_zone.rs:163
|
| 8 | 2024-05-29T21:46:43.499Z | INFO | SledAgent (BootstrapAgentStartup): finding links ["net0", "net1"]
file = sled-agent/src/bootstrap/pre_server.rs:255
|
| 9 | 2024-05-29T21:46:43.552Z | INFO | SledAgent (BootstrapAgentStartup): Starting mg-ddm service
addr-objs = [AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]
file = sled-agent/src/bootstrap/pre_server.rs:262
|
| 10 | 2024-05-29T21:46:43.552Z | INFO | SledAgent (BootstrapAgentStartup): Importing mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:42
path = /opt/oxide/mg-ddm/pkg/ddm/manifest.xml
|
| 11 | 2024-05-29T21:46:43.665Z | INFO | SledAgent (BootstrapAgentStartup): Setting mg-ddm interfaces
file = sled-agent/src/bootstrap/maghemite.rs:50
interfaces = ("net0/ll" "net1/ll")
|
| 12 | 2024-05-29T21:46:43.674Z | INFO | SledAgent (BootstrapAgentStartup): Enabling mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:56
|
| 13 | 2024-05-29T21:46:43.683Z | INFO | SledAgent (BootstrapAgentStartup): Ensuring zfs key directory exists
file = sled-agent/src/bootstrap/pre_server.rs:272
path = /var/run/oxide/
|
| 14 | 2024-05-29T21:46:44.285Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }
|
| 15 | 2024-05-29T21:46:44.427Z | INFO | SledAgent: Starting KeyManager
file = sled-agent/src/long_running_tasks.rs:119
|
| 16 | 2024-05-29T21:46:44.427Z | INFO | SledAgent: Starting StorageManager
file = sled-agent/src/long_running_tasks.rs:131
|
| 17 | 2024-05-29T21:46:44.427Z | INFO | SledAgent: Starting StorageMonitor
file = sled-agent/src/long_running_tasks.rs:141
|
| 18 | 2024-05-29T21:46:44.427Z | INFO | SledAgent: Starting HardwareManager
file = sled-agent/src/long_running_tasks.rs:162
nongimlet_observed_disks = []
sled_mode = Scrimlet { asic: SoftNpuZone }
|
| 19 | 2024-05-29T21:46:44.427Z | INFO | SledAgent (DumpSetup-worker): DumpSetup poll loop started.
file = sled-agent/src/dump_setup.rs:506
|
| 20 | 2024-05-29T21:46:44.427Z | INFO | SledAgent (HardwareManager): Creating HardwareManager
file = sled-hardware/src/illumos/mod.rs:625
|
| 21 | 2024-05-29T21:46:44.430Z | INFO | SledAgent (HardwareManager): Generated i86pc baseboard Pc { identifier: "sock", model: "i86pc" }
file = sled-hardware/src/illumos/mod.rs:535
|
| 22 | 2024-05-29T21:46:44.430Z | WARN | SledAgent (HardwareManager): Device is not a Gimlet (i86pc), proceeding with null hardware view
file = sled-hardware/src/illumos/mod.rs:674
|
| 23 | 2024-05-29T21:46:44.430Z | INFO | SledAgent: Starting HardwareMonitor
file = sled-agent/src/long_running_tasks.rs:176
|
| 24 | 2024-05-29T21:46:44.430Z | INFO | SledAgent (HardwareManager): Monitoring for hardware updates
file = sled-hardware/src/illumos/mod.rs:733
|
| 25 | 2024-05-29T21:46:44.430Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/m2_0.vdev
|
| 26 | 2024-05-29T21:46:44.430Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
disks = {}
file = sled-agent/src/hardware_monitor.rs:262
underlay_network_info = None
|
| 27 | 2024-05-29T21:46:44.430Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 28 | 2024-05-29T21:46:44.430Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 29 | 2024-05-29T21:46:44.430Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
path = /scratch/m2_0.vdev
|
| 30 | 2024-05-29T21:46:44.442Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev
file = sled-hardware/src/disk.rs:304
|
| 31 | 2024-05-29T21:46:44.442Z | INFO | SledAgent (StorageResources): Formatting zpool with generated ID
file = sled-hardware/src/disk.rs:315
id = 800d63a5-84ab-4e81-94a7-03a851147655 (zpool)
|
| 32 | 2024-05-29T21:46:44.541Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal }
|
| 33 | 2024-05-29T21:46:44.744Z | INFO | SledAgent (StorageManager): Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }
file = sled-storage/src/manager.rs:381
|
| 34 | 2024-05-29T21:46:44.744Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/m2_1.vdev
|
| 35 | 2024-05-29T21:46:44.744Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 36 | 2024-05-29T21:46:44.744Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 37 | 2024-05-29T21:46:44.744Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 38 | 2024-05-29T21:46:44.744Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 39 | 2024-05-29T21:46:44.744Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
path = /scratch/m2_1.vdev
|
| 40 | 2024-05-29T21:46:44.752Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev
file = sled-hardware/src/disk.rs:304
|
| 41 | 2024-05-29T21:46:44.752Z | INFO | SledAgent (StorageResources): Formatting zpool with generated ID
file = sled-hardware/src/disk.rs:315
id = 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool)
|
| 42 | 2024-05-29T21:46:44.841Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal }
|
| 43 | 2024-05-29T21:46:45.043Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_0.vdev
|
| 44 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 45 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 46 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 47 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 48 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 49 | 2024-05-29T21:46:45.043Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_1.vdev
|
| 50 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 51 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 52 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 53 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 54 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 55 | 2024-05-29T21:46:45.043Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_2.vdev
|
| 56 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 57 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 58 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 59 | 2024-05-29T21:46:45.043Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_3.vdev
|
| 60 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 61 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 62 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 63 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 64 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 65 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 66 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 67 | 2024-05-29T21:46:45.043Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_4.vdev
|
| 68 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 69 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 70 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 71 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 72 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 73 | 2024-05-29T21:46:45.043Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_5.vdev
|
| 74 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 75 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 76 | 2024-05-29T21:46:45.043Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_6.vdev
|
| 77 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 78 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 79 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 80 | 2024-05-29T21:46:45.043Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_7.vdev
|
| 81 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 82 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 83 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 84 | 2024-05-29T21:46:45.043Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_8.vdev
|
| 85 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 86 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
| 87 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 88 | 2024-05-29T21:46:45.043Z | INFO | SledAgent: Waiting for boot disk
file = sled-agent/src/long_running_tasks.rs:91
|
| 89 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 90 | 2024-05-29T21:46:45.043Z | INFO | SledAgent: Found boot disk DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
file = sled-agent/src/long_running_tasks.rs:93
|
| 91 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 92 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 93 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 94 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 95 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 96 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 97 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 98 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 99 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 100 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 101 | 2024-05-29T21:46:45.043Z | INFO | SledAgent: Starting Bootstore
file = sled-agent/src/long_running_tasks.rs:200
|
| 102 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 103 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 104 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (bootstore): No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 105 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (bootstore): No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 106 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (bootstore): No ledger found. Loading Fsm::State as Uninitialized
file = bootstore/src/schemes/v0/storage.rs:83
peer_id = pc-sock-i86pc
|
| 107 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (bootstore): No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/cluster/bootstore-network-config.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 108 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (bootstore): No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/cluster/bootstore-network-config.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 109 | 2024-05-29T21:46:45.043Z | INFO | SledAgent (bootstore): No ledger found for network config
file = bootstore/src/schemes/v0/storage.rs:146
peer_id = pc-sock-i86pc
|
| 110 | 2024-05-29T21:46:45.043Z | INFO | SledAgent: Starting Bootstore DDMD poller
file = sled-agent/src/long_running_tasks.rs:205
|
| 111 | 2024-05-29T21:46:45.043Z | INFO | SledAgent: Starting ZoneBundler related tasks
file = sled-agent/src/long_running_tasks.rs:220
|
| 112 | 2024-05-29T21:46:45.053Z | INFO | SledAgent (ServiceManager): Creating ServiceManager
file = sled-agent/src/services.rs:711
|
| 113 | 2024-05-29T21:46:45.053Z | INFO | SledAgent (auto-cleanup-task): top of bundle cleanup loop
file = sled-agent/src/zone_bundle.rs:289
next_cleanup = Instant { tv_sec: 1862, tv_nsec: 927767799 }
time_to_next_cleanup = 599.999941379s
|
| 114 | 2024-05-29T21:46:45.053Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3617
|
| 115 | 2024-05-29T21:46:45.053Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 116 | 2024-05-29T21:46:45.053Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/sled-agent-request.json
file = common/src/ledger.rs:188
|
| 117 | 2024-05-29T21:46:45.054Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/sled-agent-request.json
file = common/src/ledger.rs:188
|
| 118 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (dropshot (BootstrapAgent)): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:196
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
|
| 119 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (SledAgentSprocketsServer): Started listening
file = sled-agent/src/bootstrap/sprockets_server.rs:44
local_addr = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 120 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (RSS): Injecting RSS configuration: RackInitializeRequest {
trust_quorum_peers: None,
bootstrap_discovery: OnlyOurs,
ntp_servers: [
"0.pool.ntp.org",
],
dns_servers: [
1.1.1.1,
9.9.9.9,
],
internal_services_ip_pool_ranges: [
V4(
Ipv4Range {
first: 10.151.2.164,
last: 10.151.2.173,
},
),
],
external_dns_ips: [
10.151.2.164,
10.151.2.165,
],
external_dns_zone_name: "oxide.test",
external_certificates: "<redacted>",
recovery_silo: RecoverySiloConfig {
silo_name: Name(
"recovery",
),
user_name: UserId(
"recovery",
),
user_password_hash: NewPasswordHash(
PasswordHashString {
string: "$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY",
encoding: B64,
},
),
},
rack_network_config: RackNetworkConfigV1 {
rack_subnet: Ipv6Network {
addr: fd00:1122:3344:100::,
prefix: 56,
},
infra_ip_first: 10.151.2.174,
infra_ip_last: 10.151.2.174,
ports: [
PortConfigV1 {
routes: [
RouteConfig {
destination: V4(
Ipv4Network {
addr: 0.0.0.0,
prefix: 0,
},
),
nexthop: 10.151.2.100,
vlan_id: None,
},
],
addresses: [
V4(
Ipv4Network {
addr: 10.151.2.174,
prefix: 24,
},
),
],
switch: Switch0,
port: "qsfp0",
uplink_port_speed: Speed40G,
uplink_port_fec: None,
bgp_peers: [],
autoneg: false,
},
],
bgp: [],
bfd: [],
},
allowed_source_ips: Any,
}
file = sled-agent/src/rack_setup/service.rs:1003
|
| 121 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
| 122 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 123 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (RSS): No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/rss-plan-completed.marker
file = common/src/ledger.rs:188
|
| 124 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (RSS): No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/rss-plan-completed.marker
file = common/src/ledger.rs:188
|
| 125 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (RSS): RSS configuration has not been fully applied yet
file = sled-agent/src/rack_setup/service.rs:1063
|
| 126 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 127 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (RSS): No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/rss-sled-plan.json
file = common/src/ledger.rs:188
|
| 128 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (RSS): No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/rss-sled-plan.json
file = common/src/ledger.rs:188
|
| 129 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (RSS): Creating new allocation plan
file = sled-agent/src/rack_setup/service.rs:1104
|
| 130 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (RSS): Serializing plan
file = sled-agent/src/rack_setup/plan/sled.rs:118
|
| 131 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (RSS): Creating plan for the sled at fdb0:18c0:4d0d:a02a::1
file = sled-agent/src/rack_setup/plan/sled.rs:91
|
| 132 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 133 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.rss-sled-plan.json.tmp
file = common/src/ledger.rs:199
|
| 134 | 2024-05-29T21:46:45.057Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.rss-sled-plan.json.tmp
file = common/src/ledger.rs:199
|
| 135 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (RSS): Sled plan written to storage
file = sled-agent/src/rack_setup/plan/sled.rs:138
|
| 136 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (RSS): Writing Rack Network Configuration to bootstore
file = sled-agent/src/rack_setup/service.rs:1137
|
| 137 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 1, current_generation: 0
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
| 138 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 139 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 140 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 1
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-sock-i86pc
|
| 141 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (BootstrapAgentRssHandler): Received initialization request from RSS
file = sled-agent/src/bootstrap/rss_handle.rs:217
request = StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 67239fc6-3779-4bd2-ac1a-314185fe431f, rack_id: 9090f606-344e-4bff-9fcd-4f77886b6762, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 142 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (SledAgentSprocketsServer): Accepted connection
file = sled-agent/src/bootstrap/sprockets_server.rs:66
remote_addr = [fdb0:18c0:4d0d:a02a::1]:34385
|
| 143 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (SledAgentMain): Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 67239fc6-3779-4bd2-ac1a-314185fe431f, rack_id: 9090f606-344e-4bff-9fcd-4f77886b6762, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }
file = sled-agent/src/bootstrap/server.rs:360
|
| 144 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (SledAgentMain): KeyManager: using hardcoded secret retriever
file = sled-agent/src/bootstrap/server.rs:375
|
| 145 | 2024-05-29T21:46:45.058Z | INFO | SledAgent: setting up sled agent server
file = sled-agent/src/server.rs:44
|
| 146 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (StorageManager): Received KeyManagerReady
file = sled-storage/src/manager.rs:381
|
| 147 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (StorageManager): Loading storage config
file = sled-storage/src/manager.rs:530
|
| 148 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
| 149 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }
|
| 150 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = load_ledger
|
| 151 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = load_ledger
|
| 152 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (StorageManager): No ledger of physical disks exists
file = sled-storage/src/manager.rs:519
|
| 153 | 2024-05-29T21:46:45.058Z | INFO | SledAgent (StorageManager): KeyManager ready, but no ledger detected
file = sled-storage/src/manager.rs:547
|
| 154 | 2024-05-29T21:46:45.061Z | INFO | SledAgent (ServiceManager): Enabling switch zone (new)
file = sled-agent/src/services.rs:3849
|
| 155 | 2024-05-29T21:46:45.071Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 156 | 2024-05-29T21:46:45.081Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 157 | 2024-05-29T21:46:45.091Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 158 | 2024-05-29T21:46:45.100Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 159 | 2024-05-29T21:46:45.109Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 160 | 2024-05-29T21:46:45.118Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 161 | 2024-05-29T21:46:45.126Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 162 | 2024-05-29T21:46:45.134Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 163 | 2024-05-29T21:46:45.143Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 164 | 2024-05-29T21:46:45.143Z | INFO | SledAgent (StorageManager): No disks to be automatically managed
file = sled-storage/src/manager.rs:558
|
| 165 | 2024-05-29T21:46:45.143Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 166 | 2024-05-29T21:46:45.143Z | INFO | SledAgent (StorageManager): automatically managing disks
file = sled-storage/src/manager.rs:370
|
| 167 | 2024-05-29T21:46:45.143Z | INFO | SledAgent (StorageResources): Synchronizing disk managment
file = sled-storage/src/resources.rs:339
|
| 168 | 2024-05-29T21:46:45.143Z | INFO | SledAgent (StorageManager): Successfully synchronized disks without error
file = sled-storage/src/manager.rs:449
result = DisksManagementResult { status: [] }
|
| 169 | 2024-05-29T21:46:45.153Z | INFO | SledAgent: SledAgent::new(..) starting
file = sled-agent/src/sled_agent.rs:355
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 170 | 2024-05-29T21:46:45.153Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 171 | 2024-05-29T21:46:45.153Z | INFO | SledAgent: Requested swap device of size 64 GiB
file = sled-agent/src/sled_agent.rs:367
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 172 | 2024-05-29T21:46:45.153Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 173 | 2024-05-29T21:46:45.157Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 79.975379ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "66854947-8912-44eb-aaa6-5ae7111672b4", "content-length": "124", "date": "Wed, 29 May 2024 21:46:44 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "66854947-8912-44eb-aaa6-5ae7111672b4" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 174 | 2024-05-29T21:46:45.165Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:315
|
| 175 | 2024-05-29T21:46:45.167Z | INFO | SledAgent: attempting to create encrypted zvol: name="oxi_800d63a5-84ab-4e81-94a7-03a851147655/swap", size_gb=64
file = sled-agent/src/swap_device.rs:153
|
| 176 | 2024-05-29T21:46:45.183Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:354
|
| 177 | 2024-05-29T21:46:45.191Z | INFO | SledAgent: successfully created encrypted zvol: name="oxi_800d63a5-84ab-4e81-94a7-03a851147655/swap", size_gb=64
file = sled-agent/src/swap_device.rs:249
|
| 178 | 2024-05-29T21:46:45.191Z | INFO | SledAgent: adding swap device: swapname="/dev/zvol/dsk/oxi_800d63a5-84ab-4e81-94a7-03a851147655/swap"
file = sled-agent/src/swap_device.rs:96
|
| 179 | 2024-05-29T21:46:45.196Z | INFO | SledAgent: Mounting backing filesystems
file = sled-agent/src/sled_agent.rs:382
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 180 | 2024-05-29T21:46:45.196Z | INFO | SledAgent (BackingFs): Processing fmd
file = sled-agent/src/backing_fs.rs:126
|
| 181 | 2024-05-29T21:46:45.196Z | INFO | SledAgent (BackingFs): Ensuring dataset oxi_800d63a5-84ab-4e81-94a7-03a851147655/backing/fmd
file = sled-agent/src/backing_fs.rs:136
|
| 182 | 2024-05-29T21:46:45.238Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 183 | 2024-05-29T21:46:45.239Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 90.79109ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "460dd27b-a7d8-421a-9200-b82e257ee18f", "content-length": "124", "date": "Wed, 29 May 2024 21:46:44 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "460dd27b-a7d8-421a-9200-b82e257ee18f" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 184 | 2024-05-29T21:46:45.265Z | INFO | SledAgent (BackingFs): Stopping service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:170
|
| 185 | 2024-05-29T21:46:45.331Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 186 | 2024-05-29T21:46:45.332Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 215.95192ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "8d2bcbe8-5957-4c24-8944-6a9d61013f98", "content-length": "124", "date": "Wed, 29 May 2024 21:46:45 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "8d2bcbe8-5957-4c24-8944-6a9d61013f98" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 187 | 2024-05-29T21:46:45.483Z | INFO | SledAgent (BackingFs): Mounting oxi_800d63a5-84ab-4e81-94a7-03a851147655/backing/fmd on /var/fm/fmd
file = sled-agent/src/backing_fs.rs:178
|
| 188 | 2024-05-29T21:46:45.496Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/rsrc
file = sled-agent/src/backing_fs.rs:186
|
| 189 | 2024-05-29T21:46:45.496Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/ckpt
file = sled-agent/src/backing_fs.rs:186
|
| 190 | 2024-05-29T21:46:45.496Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/xprt
file = sled-agent/src/backing_fs.rs:186
|
| 191 | 2024-05-29T21:46:45.496Z | INFO | SledAgent (BackingFs): Starting service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:192
|
| 192 | 2024-05-29T21:46:45.549Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 193 | 2024-05-29T21:46:45.549Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 155.445115ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f1f7eb9a-82b1-4961-9604-414dfb43ea41", "content-length": "124", "date": "Wed, 29 May 2024 21:46:45 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f1f7eb9a-82b1-4961-9604-414dfb43ea41" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 194 | 2024-05-29T21:46:45.707Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 195 | 2024-05-29T21:46:45.708Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 116.626785ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "59ec1d8a-4e12-4ecd-b591-70790729a0d9", "content-length": "124", "date": "Wed, 29 May 2024 21:46:45 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "59ec1d8a-4e12-4ecd-b591-70790729a0d9" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 196 | 2024-05-29T21:46:45.826Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 197 | 2024-05-29T21:46:45.827Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 207.526121ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "fb0df301-97ad-4cf7-b8bb-6bb6a8bbb4de", "content-length": "124", "date": "Wed, 29 May 2024 21:46:45 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "fb0df301-97ad-4cf7-b8bb-6bb6a8bbb4de" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 198 | 2024-05-29T21:46:46.035Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 199 | 2024-05-29T21:46:46.035Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 260.859483ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "821d58d0-78da-4462-91dc-ca518b7f042d", "content-length": "124", "date": "Wed, 29 May 2024 21:46:45 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "821d58d0-78da-4462-91dc-ca518b7f042d" }
file = clients/ddm-admin-client/src/lib.rs:178
|
| 200 | 2024-05-29T21:46:46.206Z | INFO | SledAgent: using '[AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]' as data links for xde driver
file = illumos-utils/src/opte/illumos.rs:90
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 201 | 2024-05-29T21:46:46.207Z | INFO | SledAgent (dropshot): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:196
local_addr = [fd00:1122:3344:101::1]:61502
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 202 | 2024-05-29T21:46:46.207Z | INFO | SledAgent (internal-dns-resolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 203 | 2024-05-29T21:46:46.207Z | INFO | SledAgent (producer-server): starting oximeter metric producer server
address = [fd00:1122:3344:101::1]:61502
file = oximeter/producer/src/lib.rs:276
interval = 30s
producer_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 204 | 2024-05-29T21:46:46.213Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 156.495226ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:384
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 205 | 2024-05-29T21:46:46.255Z | INFO | SledAgent (VmmReservoirManager): 50% of 102952026112 physical ram = 51476013056 bytes)
file = sled-agent/src/vmm_reservoir.rs:267
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 206 | 2024-05-29T21:46:46.255Z | INFO | SledAgent (VmmReservoirManager): Setting reservoir size to 49090 MiB bytes
file = sled-agent/src/vmm_reservoir.rs:275
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 207 | 2024-05-29T21:46:46.255Z | INFO | SledAgent (ServiceManager): sled agent started
file = sled-agent/src/services.rs:993
underlay_address = fd00:1122:3344:101::1
|
| 208 | 2024-05-29T21:46:46.255Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
| 209 | 2024-05-29T21:46:46.256Z | INFO | SledAgent (dropshot (SledAgent)): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:196
local_addr = [fd00:1122:3344:101::1]:12345
|
| 210 | 2024-05-29T21:46:46.256Z | INFO | SledAgent (SledAgentMain): Sled Agent loaded; recording configuration
file = sled-agent/src/bootstrap/server.rs:430
|
| 211 | 2024-05-29T21:46:46.256Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 212 | 2024-05-29T21:46:46.256Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:199
|
| 213 | 2024-05-29T21:46:46.256Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:199
|
| 214 | 2024-05-29T21:46:46.256Z | INFO | SledAgent (HardwareMonitor): Sled Agent Started
file = sled-agent/src/hardware_monitor.rs:138
|
| 215 | 2024-05-29T21:46:46.256Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
disks = {}
file = sled-agent/src/hardware_monitor.rs:262
underlay_network_info = Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [V4(Ipv4Network { addr: 10.151.2.174, prefix: 24 })], switch: Switch0, port: "qsfp0", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))
|
| 216 | 2024-05-29T21:46:46.256Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3617
|
| 217 | 2024-05-29T21:46:46.256Z | INFO | SledAgent (SledAgentSprocketsServer): Connection closed
file = sled-agent/src/bootstrap/sprockets_server.rs:73
remote_addr = [fdb0:18c0:4d0d:a02a::1]:34385
|
| 218 | 2024-05-29T21:46:46.438Z | INFO | SledAgent (BootstrapAgentRssHandler): Peer agent initialized
file = sled-agent/src/bootstrap/rss_handle.rs:108
peer = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 219 | 2024-05-29T21:46:46.438Z | INFO | SledAgent (BootstrapAgentRssHandler): Initialized sled agent
file = sled-agent/src/bootstrap/rss_handle.rs:232
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 220 | 2024-05-29T21:46:46.439Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 221 | 2024-05-29T21:46:46.439Z | INFO | SledAgent (RSS): No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/rss-service-plan-v3.json
file = common/src/ledger.rs:188
|
| 222 | 2024-05-29T21:46:46.439Z | INFO | SledAgent (RSS): No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/rss-service-plan-v3.json
file = common/src/ledger.rs:188
|
| 223 | 2024-05-29T21:46:46.439Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 224 | 2024-05-29T21:46:46.439Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 225 | 2024-05-29T21:46:46.439Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
| 226 | 2024-05-29T21:46:46.440Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 253.465345ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:384
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 227 | 2024-05-29T21:46:46.445Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 228 | 2024-05-29T21:46:46.471Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:52632
|
| 229 | 2024-05-29T21:46:46.471Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 230 | 2024-05-29T21:46:46.471Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 152
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:52632
req_id = c8b9714e-32db-40d8-a546-955994585255
response_code = 200
uri = /inventory
|
| 231 | 2024-05-29T21:46:46.500Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:52288
|
| 232 | 2024-05-29T21:46:46.500Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 32
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:52288
req_id = 2d754a23-2fd7-4b25-ac31-4d73ce929d8e
response_code = 200
uri = /sled-role
|
| 233 | 2024-05-29T21:46:46.501Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 234 | 2024-05-29T21:46:46.501Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.rss-service-plan-v3.json.tmp
file = common/src/ledger.rs:199
|
| 235 | 2024-05-29T21:46:46.501Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.rss-service-plan-v3.json.tmp
file = common/src/ledger.rs:199
|
| 236 | 2024-05-29T21:46:46.501Z | INFO | SledAgent (RSS): Service plan written to storage
file = sled-agent/src/rack_setup/plan/service.rs:815
|
| 237 | 2024-05-29T21:46:46.530Z | INFO | SledAgent (RSS): attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: c7fac74c-ff6b-46fb-86b5-d71ea8414155, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, pool_id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool) }, OmicronPhysicalDiskConfig { id: 2943cfa4-85a5-40f3-8341-640cc8c7a118, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, pool_id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool) }, OmicronPhysicalDiskConfig { id: 0d5f0158-e44e-4ba3-bd4a-567051ccf8d1, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, pool_id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool) }, OmicronPhysicalDiskConfig { id: de483768-8ce8-4e96-9af9-1bc8a421d29d, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, pool_id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool) }, OmicronPhysicalDiskConfig { id: fd324f08-ddd2-4770-bf19-43e2485deec9, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, pool_id: 020b43a6-e481-4304-8596-25774e86f14b (zpool) }, OmicronPhysicalDiskConfig { id: 79469e5e-cd5b-404a-bedb-fb74819836c6, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, pool_id: 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool) }, OmicronPhysicalDiskConfig { id: 9a064006-65eb-4c19-8f5a-5c18c229c40c, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, pool_id: 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool) }, OmicronPhysicalDiskConfig { id: b6295793-cf00-4673-ac4a-bd13fcb081e7, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, pool_id: 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool) }, OmicronPhysicalDiskConfig { id: 21a64282-06ac-4276-8c8b-4c5f2db10bde, identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, pool_id: 6b416693-d467-424a-86a1-703b05ad3101 (zpool) }], generation: Generation(1) }
file = sled-agent/src/rack_setup/service.rs:347
sled_address = [fd00:1122:3344:101::1]:12345
|
| 238 | 2024-05-29T21:46:46.530Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:45207
|
| 239 | 2024-05-29T21:46:46.530Z | INFO | SledAgent (StorageManager): Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, id: c7fac74c-ff6b-46fb-86b5-d71ea8414155, pool_id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, id: 2943cfa4-85a5-40f3-8341-640cc8c7a118, pool_id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, id: 0d5f0158-e44e-4ba3-bd4a-567051ccf8d1, pool_id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, id: de483768-8ce8-4e96-9af9-1bc8a421d29d, pool_id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, id: fd324f08-ddd2-4770-bf19-43e2485deec9, pool_id: 020b43a6-e481-4304-8596-25774e86f14b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, id: 79469e5e-cd5b-404a-bedb-fb74819836c6, pool_id: 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, id: 9a064006-65eb-4c19-8f5a-5c18c229c40c, pool_id: 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, id: b6295793-cf00-4673-ac4a-bd13fcb081e7, pool_id: 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, id: 21a64282-06ac-4276-8c8b-4c5f2db10bde, pool_id: 6b416693-d467-424a-86a1-703b05ad3101 (zpool) }] }, tx: ... }
file = sled-storage/src/manager.rs:381
|
| 240 | 2024-05-29T21:46:46.530Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = omicron_physical_disks_ensure
|
| 241 | 2024-05-29T21:46:46.530Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = omicron_physical_disks_ensure
|
| 242 | 2024-05-29T21:46:46.530Z | INFO | SledAgent (StorageManager): No previously-stored 'requested disks', creating new ledger
file = sled-storage/src/manager.rs:681
request = omicron_physical_disks_ensure
|
| 243 | 2024-05-29T21:46:46.530Z | INFO | SledAgent (StorageResources): Synchronizing disk managment
file = sled-storage/src/resources.rs:339
|
| 244 | 2024-05-29T21:46:46.530Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:379
|
| 245 | 2024-05-29T21:46:46.530Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 246 | 2024-05-29T21:46:46.530Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
path = /scratch/u2_0.vdev
|
| 247 | 2024-05-29T21:46:46.650Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev
file = sled-hardware/src/disk.rs:304
|
| 248 | 2024-05-29T21:46:46.650Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool)
|
| 249 | 2024-05-29T21:46:46.827Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 549.653104ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:384
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 250 | 2024-05-29T21:46:47.258Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 251 | 2024-05-29T21:46:47.445Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 2.435479357s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:384
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 252 | 2024-05-29T21:46:47.636Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }
|
| 253 | 2024-05-29T21:46:47.810Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:332
|
| 254 | 2024-05-29T21:46:47.810Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 255 | 2024-05-29T21:46:47.810Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 256 | 2024-05-29T21:46:47.810Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 257 | 2024-05-29T21:46:47.810Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 258 | 2024-05-29T21:46:47.810Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 259 | 2024-05-29T21:46:47.810Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 260 | 2024-05-29T21:46:48.400Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_switch
|
| 261 | 2024-05-29T21:46:48.632Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 262 | 2024-05-29T21:46:48.632Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 263 | 2024-05-29T21:46:49.043Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 264 | 2024-05-29T21:46:49.043Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 265 | 2024-05-29T21:46:49.221Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 42.089567ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 266 | 2024-05-29T21:46:49.587Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 134.829784ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 267 | 2024-05-29T21:46:50.025Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 268 | 2024-05-29T21:46:50.025Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 3.601439735s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:384
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 269 | 2024-05-29T21:46:50.268Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 182.495348ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 270 | 2024-05-29T21:46:50.947Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 438.164099ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 271 | 2024-05-29T21:46:50.949Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 272 | 2024-05-29T21:46:51.157Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 273 | 2024-05-29T21:46:51.614Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 530.350853ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 274 | 2024-05-29T21:46:52.106Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c
file = sled-storage/src/dataset.rs:480
|
| 275 | 2024-05-29T21:46:52.548Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 276 | 2024-05-29T21:46:52.548Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:477
|
| 277 | 2024-05-29T21:46:52.548Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:394
|
| 278 | 2024-05-29T21:46:52.548Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:379
|
| 279 | 2024-05-29T21:46:52.548Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 280 | 2024-05-29T21:46:52.548Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
path = /scratch/u2_1.vdev
|
| 281 | 2024-05-29T21:46:52.772Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 1.145327903s
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 282 | 2024-05-29T21:46:52.774Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 283 | 2024-05-29T21:46:52.781Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev
file = sled-hardware/src/disk.rs:304
|
| 284 | 2024-05-29T21:46:52.781Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 8f301520-a970-47c3-90c3-bd3006e5615c (zpool)
|
| 285 | 2024-05-29T21:46:53.738Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 286 | 2024-05-29T21:46:53.738Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 10.581601903s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:384
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 287 | 2024-05-29T21:46:54.251Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }
|
| 288 | 2024-05-29T21:46:54.722Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:332
|
| 289 | 2024-05-29T21:46:54.722Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 290 | 2024-05-29T21:46:54.722Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 291 | 2024-05-29T21:46:54.722Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 292 | 2024-05-29T21:46:54.722Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 293 | 2024-05-29T21:46:54.722Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 294 | 2024-05-29T21:46:54.722Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 295 | 2024-05-29T21:46:55.718Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 296 | 2024-05-29T21:46:55.718Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 297 | 2024-05-29T21:46:56.246Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 298 | 2024-05-29T21:46:56.478Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_8f301520-a970-47c3-90c3-bd3006e5615c/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 299 | 2024-05-29T21:46:56.923Z | INFO | SledAgent (VmmReservoirManager): Finished setting reservoir size to 49090 MiB bytes
file = sled-agent/src/vmm_reservoir.rs:279
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 300 | 2024-05-29T21:46:56.925Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 301 | 2024-05-29T21:46:57.042Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_8f301520-a970-47c3-90c3-bd3006e5615c
file = sled-storage/src/dataset.rs:480
|
| 302 | 2024-05-29T21:46:57.050Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:477
|
| 303 | 2024-05-29T21:46:57.050Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:394
|
| 304 | 2024-05-29T21:46:57.050Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:379
|
| 305 | 2024-05-29T21:46:57.050Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 306 | 2024-05-29T21:46:57.050Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
path = /scratch/u2_2.vdev
|
| 307 | 2024-05-29T21:46:57.059Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev
file = sled-hardware/src/disk.rs:304
|
| 308 | 2024-05-29T21:46:57.059Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool)
|
| 309 | 2024-05-29T21:46:57.155Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }
|
| 310 | 2024-05-29T21:46:57.164Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/dataset.rs:332
|
| 311 | 2024-05-29T21:46:57.164Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 312 | 2024-05-29T21:46:57.164Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 313 | 2024-05-29T21:46:57.164Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 314 | 2024-05-29T21:46:57.164Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 315 | 2024-05-29T21:46:57.164Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 316 | 2024-05-29T21:46:57.164Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 317 | 2024-05-29T21:46:57.199Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 318 | 2024-05-29T21:46:57.206Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 319 | 2024-05-29T21:46:57.226Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 320 | 2024-05-29T21:46:57.328Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff
file = sled-storage/src/dataset.rs:480
|
| 321 | 2024-05-29T21:46:57.336Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:477
|
| 322 | 2024-05-29T21:46:57.336Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:394
|
| 323 | 2024-05-29T21:46:57.336Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:379
|
| 324 | 2024-05-29T21:46:57.336Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 325 | 2024-05-29T21:46:57.336Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
path = /scratch/u2_3.vdev
|
| 326 | 2024-05-29T21:46:57.346Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev
file = sled-hardware/src/disk.rs:304
|
| 327 | 2024-05-29T21:46:57.346Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool)
|
| 328 | 2024-05-29T21:46:57.442Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }
|
| 329 | 2024-05-29T21:46:57.450Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/dataset.rs:332
|
| 330 | 2024-05-29T21:46:57.450Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 331 | 2024-05-29T21:46:57.450Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 332 | 2024-05-29T21:46:57.450Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 333 | 2024-05-29T21:46:57.450Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 334 | 2024-05-29T21:46:57.450Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 335 | 2024-05-29T21:46:57.450Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 336 | 2024-05-29T21:46:57.486Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 337 | 2024-05-29T21:46:57.495Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_b5814935-58c2-4e3c-9137-4cbaa1ee9fab/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 338 | 2024-05-29T21:46:57.618Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_b5814935-58c2-4e3c-9137-4cbaa1ee9fab
file = sled-storage/src/dataset.rs:480
|
| 339 | 2024-05-29T21:46:57.626Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:477
|
| 340 | 2024-05-29T21:46:57.626Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:394
|
| 341 | 2024-05-29T21:46:57.626Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:379
|
| 342 | 2024-05-29T21:46:57.626Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 343 | 2024-05-29T21:46:57.626Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
path = /scratch/u2_4.vdev
|
| 344 | 2024-05-29T21:46:57.635Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev
file = sled-hardware/src/disk.rs:304
|
| 345 | 2024-05-29T21:46:57.635Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 020b43a6-e481-4304-8596-25774e86f14b (zpool)
|
| 346 | 2024-05-29T21:46:57.730Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }
|
| 347 | 2024-05-29T21:46:57.738Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/dataset.rs:332
|
| 348 | 2024-05-29T21:46:57.738Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 349 | 2024-05-29T21:46:57.738Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 350 | 2024-05-29T21:46:57.738Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 351 | 2024-05-29T21:46:57.738Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 352 | 2024-05-29T21:46:57.738Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 353 | 2024-05-29T21:46:57.738Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 354 | 2024-05-29T21:46:57.773Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 355 | 2024-05-29T21:46:57.781Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_020b43a6-e481-4304-8596-25774e86f14b/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 356 | 2024-05-29T21:46:57.906Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_020b43a6-e481-4304-8596-25774e86f14b
file = sled-storage/src/dataset.rs:480
|
| 357 | 2024-05-29T21:46:57.914Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:477
|
| 358 | 2024-05-29T21:46:57.914Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:394
|
| 359 | 2024-05-29T21:46:57.914Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:379
|
| 360 | 2024-05-29T21:46:57.914Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 361 | 2024-05-29T21:46:57.915Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
path = /scratch/u2_5.vdev
|
| 362 | 2024-05-29T21:46:57.925Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev
file = sled-hardware/src/disk.rs:304
|
| 363 | 2024-05-29T21:46:57.925Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool)
|
| 364 | 2024-05-29T21:46:58.017Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool), kind: External }
|
| 365 | 2024-05-29T21:46:58.025Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/dataset.rs:332
|
| 366 | 2024-05-29T21:46:58.025Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 367 | 2024-05-29T21:46:58.025Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 368 | 2024-05-29T21:46:58.025Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 369 | 2024-05-29T21:46:58.025Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 370 | 2024-05-29T21:46:58.026Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 371 | 2024-05-29T21:46:58.026Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 372 | 2024-05-29T21:46:58.029Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 373 | 2024-05-29T21:46:58.061Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 374 | 2024-05-29T21:46:58.066Z | INFO | SledAgent (ServiceManager): Ensuring tfportrear0_0/ll exists in zone
file = sled-agent/src/services.rs:2303
|
| 375 | 2024-05-29T21:46:58.070Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_4532d18b-4909-4537-9ce6-98b74fb663f5/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 376 | 2024-05-29T21:46:58.195Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_4532d18b-4909-4537-9ce6-98b74fb663f5
file = sled-storage/src/dataset.rs:480
|
| 377 | 2024-05-29T21:46:58.203Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:477
|
| 378 | 2024-05-29T21:46:58.203Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:394
|
| 379 | 2024-05-29T21:46:58.203Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:379
|
| 380 | 2024-05-29T21:46:58.203Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 381 | 2024-05-29T21:46:58.203Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
path = /scratch/u2_6.vdev
|
| 382 | 2024-05-29T21:46:58.213Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev
file = sled-hardware/src/disk.rs:304
|
| 383 | 2024-05-29T21:46:58.213Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool)
|
| 384 | 2024-05-29T21:46:58.306Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool), kind: External }
|
| 385 | 2024-05-29T21:46:58.314Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/dataset.rs:332
|
| 386 | 2024-05-29T21:46:58.314Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 387 | 2024-05-29T21:46:58.314Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 388 | 2024-05-29T21:46:58.314Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 389 | 2024-05-29T21:46:58.314Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 390 | 2024-05-29T21:46:58.314Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 391 | 2024-05-29T21:46:58.314Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 392 | 2024-05-29T21:46:58.349Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 393 | 2024-05-29T21:46:58.358Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_8d81843f-0d2d-4c69-8d17-45004d7cf484/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 394 | 2024-05-29T21:46:58.481Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_8d81843f-0d2d-4c69-8d17-45004d7cf484
file = sled-storage/src/dataset.rs:480
|
| 395 | 2024-05-29T21:46:58.489Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:477
|
| 396 | 2024-05-29T21:46:58.490Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:394
|
| 397 | 2024-05-29T21:46:58.490Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:379
|
| 398 | 2024-05-29T21:46:58.490Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 399 | 2024-05-29T21:46:58.490Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
path = /scratch/u2_7.vdev
|
| 400 | 2024-05-29T21:46:58.500Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev
file = sled-hardware/src/disk.rs:304
|
| 401 | 2024-05-29T21:46:58.500Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool)
|
| 402 | 2024-05-29T21:46:58.600Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool), kind: External }
|
| 403 | 2024-05-29T21:46:58.608Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/dataset.rs:332
|
| 404 | 2024-05-29T21:46:58.608Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 405 | 2024-05-29T21:46:58.608Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 406 | 2024-05-29T21:46:58.608Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 407 | 2024-05-29T21:46:58.608Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 408 | 2024-05-29T21:46:58.608Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 409 | 2024-05-29T21:46:58.608Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 410 | 2024-05-29T21:46:58.642Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 411 | 2024-05-29T21:46:58.650Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_59fcaf06-997c-43d9-9256-26a5eda3000f/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 412 | 2024-05-29T21:46:58.771Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_59fcaf06-997c-43d9-9256-26a5eda3000f
file = sled-storage/src/dataset.rs:480
|
| 413 | 2024-05-29T21:46:58.779Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:477
|
| 414 | 2024-05-29T21:46:58.779Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:394
|
| 415 | 2024-05-29T21:46:58.779Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:379
|
| 416 | 2024-05-29T21:46:58.779Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
| 417 | 2024-05-29T21:46:58.779Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:121
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
path = /scratch/u2_8.vdev
|
| 418 | 2024-05-29T21:46:58.789Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev
file = sled-hardware/src/disk.rs:304
|
| 419 | 2024-05-29T21:46:58.789Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 6b416693-d467-424a-86a1-703b05ad3101 (zpool)
|
| 420 | 2024-05-29T21:46:58.888Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 6b416693-d467-424a-86a1-703b05ad3101 (zpool), kind: External }
|
| 421 | 2024-05-29T21:46:58.895Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/dataset.rs:332
|
| 422 | 2024-05-29T21:46:58.896Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
| 423 | 2024-05-29T21:46:58.896Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
| 424 | 2024-05-29T21:46:58.896Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
| 425 | 2024-05-29T21:46:58.896Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 426 | 2024-05-29T21:46:58.896Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 427 | 2024-05-29T21:46:58.896Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
| 428 | 2024-05-29T21:46:58.932Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 429 | 2024-05-29T21:46:58.940Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_6b416693-d467-424a-86a1-703b05ad3101/crypt/zone
file = sled-storage/src/dataset.rs:395
|
| 430 | 2024-05-29T21:46:59.032Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 431 | 2024-05-29T21:46:59.060Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_6b416693-d467-424a-86a1-703b05ad3101
file = sled-storage/src/dataset.rs:480
|
| 432 | 2024-05-29T21:46:59.068Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:477
|
| 433 | 2024-05-29T21:46:59.068Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:394
|
| 434 | 2024-05-29T21:46:59.068Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.omicron-physical-disks.json.tmp
file = common/src/ledger.rs:199
request = omicron_physical_disks_ensure
|
| 435 | 2024-05-29T21:46:59.068Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 6b416693-d467-424a-86a1-703b05ad3101 (zpool), kind: External } }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
| 436 | 2024-05-29T21:46:59.068Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
| 437 | 2024-05-29T21:46:59.068Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 438 | 2024-05-29T21:46:59.068Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.omicron-physical-disks.json.tmp
file = common/src/ledger.rs:199
request = omicron_physical_disks_ensure
|
| 439 | 2024-05-29T21:46:59.069Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 12539057
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:45207
req_id = 6a4c3716-dd5a-48bd-8ebe-bb6cb8d3ca3c
response_code = 200
uri = /omicron-physical-disks
|
| 440 | 2024-05-29T21:46:59.099Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 441 | 2024-05-29T21:46:59.100Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:48468
|
| 442 | 2024-05-29T21:46:59.100Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 443 | 2024-05-29T21:46:59.100Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 444 | 2024-05-29T21:46:59.150Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 445 | 2024-05-29T21:46:59.150Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 446 | 2024-05-29T21:46:59.200Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 447 | 2024-05-29T21:46:59.200Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 448 | 2024-05-29T21:46:59.223Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 449 | 2024-05-29T21:46:59.248Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 450 | 2024-05-29T21:46:59.248Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 451 | 2024-05-29T21:46:59.248Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 452 | 2024-05-29T21:46:59.248Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 453 | 2024-05-29T21:46:59.290Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 454 | 2024-05-29T21:46:59.290Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 455 | 2024-05-29T21:46:59.290Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 456 | 2024-05-29T21:46:59.335Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f
file = illumos-utils/src/zone.rs:315
|
| 457 | 2024-05-29T21:46:59.339Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3
file = illumos-utils/src/zone.rs:315
|
| 458 | 2024-05-29T21:46:59.343Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3
file = illumos-utils/src/zone.rs:315
|
| 459 | 2024-05-29T21:46:59.356Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f
file = illumos-utils/src/zone.rs:354
|
| 460 | 2024-05-29T21:46:59.361Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3
file = illumos-utils/src/zone.rs:354
|
| 461 | 2024-05-29T21:46:59.364Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3
file = illumos-utils/src/zone.rs:354
|
| 462 | 2024-05-29T21:47:00.034Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 463 | 2024-05-29T21:47:00.885Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService1"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:1::2"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:1::1"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:1::1]:5353"/>
<propval type="astring" name="dns_address" value="[fd00:1122:3344:1::1]:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 464 | 2024-05-29T21:47:00.885Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }
|
| 465 | 2024-05-29T21:47:00.885Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f
|
| 466 | 2024-05-29T21:47:01.367Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService2"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:3::2"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:3::1"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:3::1]:5353"/>
<propval type="astring" name="dns_address" value="[fd00:1122:3344:3::1]:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 467 | 2024-05-29T21:47:01.847Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }
|
| 468 | 2024-05-29T21:47:01.847Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService3"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:2::2"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:2::1"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:2::1]:5353"/>
<propval type="astring" name="dns_address" value="[fd00:1122:3344:2::1]:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 469 | 2024-05-29T21:47:01.847Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }
|
| 470 | 2024-05-29T21:47:01.856Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f") failed: Property not found. retry in 47.089157ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f
|
| 471 | 2024-05-29T21:47:01.856Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3
|
| 472 | 2024-05-29T21:47:01.859Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3
|
| 473 | 2024-05-29T21:47:01.912Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f") failed: Property not found. retry in 64.780637ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f
|
| 474 | 2024-05-29T21:47:01.986Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f") failed: Property not found. retry in 293.85354ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f
|
| 475 | 2024-05-29T21:47:01.991Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3") failed: Property not found. retry in 59.610186ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3
|
| 476 | 2024-05-29T21:47:01.996Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3") failed: Property not found. retry in 25.762739ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3
|
| 477 | 2024-05-29T21:47:02.030Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3") failed: Property not found. retry in 65.361389ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3
|
| 478 | 2024-05-29T21:47:02.234Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3") failed: Property not found. retry in 145.093193ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3
|
| 479 | 2024-05-29T21:47:02.269Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3") failed: Property not found. retry in 250.961638ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3
|
| 480 | 2024-05-29T21:47:02.309Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f") failed: Property not found. retry in 412.486754ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f
|
| 481 | 2024-05-29T21:47:02.404Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3") failed: Property not found. retry in 130.48279ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3
|
| 482 | 2024-05-29T21:47:02.531Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3") failed: Property not found. retry in 275.167442ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3
|
| 483 | 2024-05-29T21:47:02.544Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3") failed: Property not found. retry in 520.071929ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3
|
| 484 | 2024-05-29T21:47:02.731Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f") failed: Property not found. retry in 818.293165ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f
|
| 485 | 2024-05-29T21:47:02.815Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3") failed: Property not found. retry in 825.82974ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3
|
| 486 | 2024-05-29T21:47:03.074Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3") failed: Property not found. retry in 445.627231ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3
|
| 487 | 2024-05-29T21:47:03.531Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3") failed: Property not found. retry in 603.041982ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3
|
| 488 | 2024-05-29T21:47:03.559Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f") failed: Property not found. retry in 682.067461ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f
|
| 489 | 2024-05-29T21:47:03.651Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3") failed: Property not found. retry in 608.04964ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3
|
| 490 | 2024-05-29T21:47:04.143Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3") failed: Property not found. retry in 937.656224ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3
|
| 491 | 2024-05-29T21:47:04.248Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f") failed: Property not found. retry in 1.413476663s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f
|
| 492 | 2024-05-29T21:47:04.266Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3") failed: Property not found. retry in 1.223968356s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3
|
| 493 | 2024-05-29T21:47:04.273Z | INFO | SledAgent (ServiceManager): Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone
file = sled-agent/src/services.rs:2320
|
| 494 | 2024-05-29T21:47:04.273Z | INFO | SledAgent (ServiceManager): Adding bootstrap address
file = illumos-utils/src/running_zone.rs:626
zone = oxz_switch
|
| 495 | 2024-05-29T21:47:04.818Z | INFO | SledAgent (ServiceManager): Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe0f:a3ef
file = sled-agent/src/services.rs:2327
|
| 496 | 2024-05-29T21:47:04.824Z | INFO | SledAgent (ServiceManager): Zone not using gateway (even though underlay is up)
file = sled-agent/src/services.rs:2388
|
| 497 | 2024-05-29T21:47:04.944Z | INFO | SledAgent (ServiceManager): Setting up dendrite service
file = sled-agent/src/services.rs:2573
|
| 498 | 2024-05-29T21:47:05.091Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3") failed: Property not found. retry in 955.826306ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3
|
| 499 | 2024-05-29T21:47:05.184Z | INFO | SledAgent (ServiceManager): Setting up lldpd service
file = sled-agent/src/services.rs:2737
|
| 500 | 2024-05-29T21:47:05.370Z | INFO | SledAgent (ServiceManager): Setting up MGS service
file = sled-agent/src/services.rs:2452
|
| 501 | 2024-05-29T21:47:05.500Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3") failed: Property not found. retry in 1.167319048s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3
|
| 502 | 2024-05-29T21:47:05.665Z | INFO | SledAgent (ServiceManager): Setting up wicketd service
file = sled-agent/src/services.rs:2487
|
| 503 | 2024-05-29T21:47:05.671Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f") failed: Property not found. retry in 1.485472686s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f
|
| 504 | 2024-05-29T21:47:05.874Z | INFO | SledAgent (ServiceManager): Setting up mgd service
file = sled-agent/src/services.rs:2787
|
| 505 | 2024-05-29T21:47:06.057Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3") failed: Property not found. retry in 1.311145227s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3
|
| 506 | 2024-05-29T21:47:06.061Z | INFO | SledAgent (ServiceManager): Setting up mg-ddm service
file = sled-agent/src/services.rs:2814
|
| 507 | 2024-05-29T21:47:06.282Z | INFO | SledAgent (ServiceManager): Setting up tfport service
file = sled-agent/src/services.rs:2681
|
| 508 | 2024-05-29T21:47:06.465Z | INFO | SledAgent (ServiceManager): Setting up Simulated SP service
file = sled-agent/src/services.rs:2481
|
| 509 | 2024-05-29T21:47:06.475Z | INFO | SledAgent (ServiceManager): Re-enabling running switch zone (new address)
file = sled-agent/src/services.rs:3873
new = [fd00:1122:3344:101::2, ::1]
old = [::1]
|
| 510 | 2024-05-29T21:47:06.475Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists
file = sled-agent/src/services.rs:3889
|
| 511 | 2024-05-29T21:47:06.475Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:609
zone = oxz_switch
|
| 512 | 2024-05-29T21:47:06.676Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3") failed: Property not found. retry in 769.733382ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3
|
| 513 | 2024-05-29T21:47:07.034Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists - OK
file = sled-agent/src/services.rs:3897
|
| 514 | 2024-05-29T21:47:07.041Z | INFO | SledAgent (ServiceManager): configuring dendrite service
file = sled-agent/src/services.rs:3953
|
| 515 | 2024-05-29T21:47:07.207Z | INFO | SledAgent (ServiceManager): configuring lldp service
file = sled-agent/src/services.rs:4015
|
| 516 | 2024-05-29T21:47:07.314Z | INFO | SledAgent (ServiceManager): configuring wicketd
file = sled-agent/src/services.rs:3996
rack_subnet = fd00:1122:3344::
|
| 517 | 2024-05-29T21:47:07.336Z | INFO | SledAgent (ServiceManager): configuring mgd service
file = sled-agent/src/services.rs:4043
|
| 518 | 2024-05-29T21:47:07.436Z | INFO | SledAgent (ServiceManager): configuring mg-ddm service
file = sled-agent/src/services.rs:4070
|
| 519 | 2024-05-29T21:47:07.467Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f
|
| 520 | 2024-05-29T21:47:07.467Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3
|
| 521 | 2024-05-29T21:47:07.467Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3
|
| 522 | 2024-05-29T21:47:07.467Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 523 | 2024-05-29T21:47:07.467Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 524 | 2024-05-29T21:47:07.467Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 8367853
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:48468
req_id = 132cb89d-8ae9-4d55-86b7-0c869870847d
response_code = 204
uri = /omicron-zones
|
| 525 | 2024-05-29T21:47:07.469Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:1::1]:5353
file = sled-agent/src/rack_setup/service.rs:548
|
| 526 | 2024-05-29T21:47:07.530Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:2::1]:5353
file = sled-agent/src/rack_setup/service.rs:548
|
| 527 | 2024-05-29T21:47:07.567Z | INFO | SledAgent (ServiceManager): Determining physical location of our switch zone at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:365
|
| 528 | 2024-05-29T21:47:07.602Z | INFO | SledAgent (ServiceManager): Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:413
|
| 529 | 2024-05-29T21:47:07.634Z | INFO | SledAgent (ServiceManager): Building Port Configuration
file = sled-agent/src/bootstrap/early_networking.rs:651
|
| 530 | 2024-05-29T21:47:07.634Z | INFO | SledAgent (ServiceManager): Checking dendrite uptime
file = sled-agent/src/bootstrap/early_networking.rs:689
|
| 531 | 2024-05-29T21:47:07.634Z | INFO | SledAgent (ServiceManager): Dendrite online
file = sled-agent/src/bootstrap/early_networking.rs:692
uptime = 2
|
| 532 | 2024-05-29T21:47:07.634Z | INFO | SledAgent (ServiceManager): Configuring default uplink on switch
config = PortSettings {\n links: {\n "0": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}
file = sled-agent/src/bootstrap/early_networking.rs:435
|
| 533 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageManager): Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }
file = sled-storage/src/manager.rs:381
|
| 534 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 535 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 536 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 537 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 538 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 539 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 540 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 541 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 542 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 543 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 544 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 545 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 546 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 547 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 548 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 549 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 550 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 551 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 552 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 553 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 554 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 555 | 2024-05-29T21:47:07.709Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 556 | 2024-05-29T21:47:07.721Z | WARN | SledAgent (RSS): failed to write DNS configuration (will retry in 69.51613ms)
dns_config_addr = [fd00:1122:3344:2::1]:5353
error_message = Communication Error: error sending request for url (http://[fd00:1122:3344:2::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:568
|
| 557 | 2024-05-29T21:47:07.811Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:3::1]:5353
file = sled-agent/src/rack_setup/service.rs:548
|
| 558 | 2024-05-29T21:47:07.860Z | INFO | SledAgent (RSS): Configured all DNS servers
file = sled-agent/src/rack_setup/service.rs:584
|
| 559 | 2024-05-29T21:47:07.860Z | INFO | SledAgent (RSS): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:259
|
| 560 | 2024-05-29T21:47:07.891Z | INFO | SledAgent (RSS): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:276
|
| 561 | 2024-05-29T21:47:07.892Z | INFO | SledAgent (RSS): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:301
|
| 562 | 2024-05-29T21:47:07.892Z | INFO | SledAgent (RSS): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:193
|
| 563 | 2024-05-29T21:47:07.920Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 564 | 2024-05-29T21:47:07.920Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:39528
|
| 565 | 2024-05-29T21:47:07.921Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 566 | 2024-05-29T21:47:07.921Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 567 | 2024-05-29T21:47:07.942Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 568 | 2024-05-29T21:47:07.942Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 569 | 2024-05-29T21:47:07.959Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 570 | 2024-05-29T21:47:07.959Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 571 | 2024-05-29T21:47:07.978Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 572 | 2024-05-29T21:47:07.978Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 573 | 2024-05-29T21:47:07.978Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 574 | 2024-05-29T21:47:07.978Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 575 | 2024-05-29T21:47:07.989Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:259
|
| 576 | 2024-05-29T21:47:08.022Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:276
|
| 577 | 2024-05-29T21:47:08.022Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:301
|
| 578 | 2024-05-29T21:47:08.022Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:193
|
| 579 | 2024-05-29T21:47:08.065Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte0", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 187, 96]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: "vopte0" } }
|
| 580 | 2024-05-29T21:47:08.065Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = ntp
|
| 581 | 2024-05-29T21:47:08.066Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 582 | 2024-05-29T21:47:08.066Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 583 | 2024-05-29T21:47:08.066Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 584 | 2024-05-29T21:47:08.069Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 585 | 2024-05-29T21:47:08.093Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575
file = illumos-utils/src/zone.rs:315
|
| 586 | 2024-05-29T21:47:08.111Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575
file = illumos-utils/src/zone.rs:354
|
| 587 | 2024-05-29T21:47:08.728Z | INFO | SledAgent (ServiceManager): Profile for oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService4"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::1b"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/chrony-setup">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="allow" value="fd00:1122:3344:100::/56"/>
<propval type="boolean" name="boundary" value="true"/>
<propval type="astring" name="server" value="0.pool.ntp.org"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="1.1.1.1" />
<value_node value="9.9.9.9" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/ntp">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte0"/>
<propval type="astring" name="gateway" value="172.30.3.1"/>
<propval type="astring" name="ip" value="172.30.3.5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 588 | 2024-05-29T21:47:08.728Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575
|
| 589 | 2024-05-29T21:47:08.865Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575") failed: Property not found. retry in 74.483841ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575
|
| 590 | 2024-05-29T21:47:08.950Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575") failed: Property not found. retry in 143.814583ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575
|
| 591 | 2024-05-29T21:47:09.107Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575") failed: Property not found. retry in 162.761024ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575
|
| 592 | 2024-05-29T21:47:09.279Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575") failed: Property not found. retry in 416.494176ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575
|
| 593 | 2024-05-29T21:47:09.705Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575") failed: Property not found. retry in 1.026561997s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575
|
| 594 | 2024-05-29T21:47:10.742Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575") failed: Property not found. retry in 532.969443ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575
|
| 595 | 2024-05-29T21:47:10.923Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45087
|
| 596 | 2024-05-29T21:47:10.924Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 873
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45087
req_id = 6f159c2f-5465-4cce-b080-f4853deff40a
response_code = 200
uri = /baseboard
|
| 597 | 2024-05-29T21:47:11.284Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575") failed: Property not found. retry in 1.088555209s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575
|
| 598 | 2024-05-29T21:47:12.381Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575") failed: Property not found. retry in 1.485682999s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575
|
| 599 | 2024-05-29T21:47:13.886Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575
|
| 600 | 2024-05-29T21:47:13.887Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 601 | 2024-05-29T21:47:13.887Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 602 | 2024-05-29T21:47:13.887Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 5966648
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:39528
req_id = 19063087-a9c7-4d43-8f03-cbaf56ec0097
response_code = 204
uri = /omicron-zones
|
| 603 | 2024-05-29T21:47:13.889Z | INFO | SledAgent (RSS): Waiting for rack time synchronization
file = sled-agent/src/rack_setup/service.rs:625
|
| 604 | 2024-05-29T21:47:13.918Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
| 605 | 2024-05-29T21:47:13.918Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:38320
|
| 606 | 2024-05-29T21:47:13.932Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3603
|
| 607 | 2024-05-29T21:47:13.932Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 13847
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:38320
req_id = a9edbd9c-14ab-4e25-af9a-b483963a2200
response_code = 500
uri = /timesync
|
| 608 | 2024-05-29T21:47:13.933Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
| 609 | 2024-05-29T21:47:14.101Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
| 610 | 2024-05-29T21:47:14.102Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:62549
|
| 611 | 2024-05-29T21:47:18.111Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3603
|
| 612 | 2024-05-29T21:47:18.111Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 4008999
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:62549
req_id = 71db486d-b221-4514-96d1-55a7fe7d78a4
response_code = 500
uri = /timesync
|
| 613 | 2024-05-29T21:47:18.114Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
| 614 | 2024-05-29T21:47:18.284Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
| 615 | 2024-05-29T21:47:18.284Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:34547
|
| 616 | 2024-05-29T21:47:18.292Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3603
|
| 617 | 2024-05-29T21:47:18.292Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 8304
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:34547
req_id = f2c74171-29e6-4a47-a896-85cb3711cbcd
response_code = 500
uri = /timesync
|
| 618 | 2024-05-29T21:47:18.294Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
| 619 | 2024-05-29T21:47:18.431Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
| 620 | 2024-05-29T21:47:18.431Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:51786
|
| 621 | 2024-05-29T21:47:22.439Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3603
|
| 622 | 2024-05-29T21:47:22.440Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 4008558
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:51786
req_id = 601d49e5-7d26-49a2-b83a-0bf97750eedb
response_code = 500
uri = /timesync
|
| 623 | 2024-05-29T21:47:22.442Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
| 624 | 2024-05-29T21:47:22.638Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
| 625 | 2024-05-29T21:47:22.638Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:52605
|
| 626 | 2024-05-29T21:47:22.646Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 8474
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:52605
req_id = 62191532-fea6-4c1c-980f-618cf4fa55f3
response_code = 200
uri = /timesync
|
| 627 | 2024-05-29T21:47:22.649Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
| 628 | 2024-05-29T21:47:22.649Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
| 629 | 2024-05-29T21:47:22.854Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
| 630 | 2024-05-29T21:47:22.854Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:58746
|
| 631 | 2024-05-29T21:47:22.862Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 8111
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:58746
req_id = bc291883-6e9a-47fc-bb8c-378f7f33f9a1
response_code = 200
uri = /timesync
|
| 632 | 2024-05-29T21:47:22.865Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
| 633 | 2024-05-29T21:47:22.865Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
| 634 | 2024-05-29T21:47:23.164Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
| 635 | 2024-05-29T21:47:23.165Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:57872
|
| 636 | 2024-05-29T21:47:23.173Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 7884
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:57872
req_id = be309704-6729-47c9-a583-f53785cf5763
response_code = 200
uri = /timesync
|
| 637 | 2024-05-29T21:47:23.175Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
| 638 | 2024-05-29T21:47:23.175Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
| 639 | 2024-05-29T21:47:23.393Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
| 640 | 2024-05-29T21:47:23.394Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:54372
|
| 641 | 2024-05-29T21:47:23.401Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 7593
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:54372
req_id = 939e5c05-1356-4bec-a5bc-1c14fa6f5402
response_code = 200
uri = /timesync
|
| 642 | 2024-05-29T21:47:23.403Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
| 643 | 2024-05-29T21:47:23.403Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
| 644 | 2024-05-29T21:47:23.796Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
| 645 | 2024-05-29T21:47:23.796Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:63019
|
| 646 | 2024-05-29T21:47:23.803Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 7585
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:63019
req_id = bf1b74f6-0c3b-4211-b06a-639929e3c6c0
response_code = 200
uri = /timesync
|
| 647 | 2024-05-29T21:47:23.806Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
| 648 | 2024-05-29T21:47:23.806Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
| 649 | 2024-05-29T21:47:24.128Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
| 650 | 2024-05-29T21:47:24.128Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:54709
|
| 651 | 2024-05-29T21:47:24.136Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 7941
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:54709
req_id = f1bdcd06-d2af-4e7a-b6e8-8718b7a369c7
response_code = 200
uri = /timesync
|
| 652 | 2024-05-29T21:47:24.138Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
| 653 | 2024-05-29T21:47:24.138Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
| 654 | 2024-05-29T21:47:24.362Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 304.205075ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:417
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 655 | 2024-05-29T21:47:25.252Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
| 656 | 2024-05-29T21:47:25.252Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:37293
|
| 657 | 2024-05-29T21:47:25.260Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 8276
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:37293
req_id = e5dfa1c5-ee8a-4dbd-9e13-615a16ddb68c
response_code = 200
uri = /timesync
|
| 658 | 2024-05-29T21:47:25.263Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
| 659 | 2024-05-29T21:47:25.263Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
| 660 | 2024-05-29T21:47:25.020Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
| 661 | 2024-05-29T21:47:25.020Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:57107
|
| 662 | 2024-05-29T21:47:25.041Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 20304
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:57107
req_id = 0f72aecd-27f7-4218-b67a-cee95faf63d9
response_code = 200
uri = /timesync
|
| 663 | 2024-05-29T21:47:25.043Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 3638887493, ip_addr: 216.229.4.69, stratum: 3, ref_time: 1717019244.7225652, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
| 664 | 2024-05-29T21:47:25.043Z | INFO | SledAgent (RSS): Finished setting up Internal DNS and NTP
file = sled-agent/src/rack_setup/service.rs:1216
|
| 665 | 2024-05-29T21:47:25.072Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 666 | 2024-05-29T21:47:25.072Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:43427
|
| 667 | 2024-05-29T21:47:25.072Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 668 | 2024-05-29T21:47:25.072Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 669 | 2024-05-29T21:47:25.092Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 670 | 2024-05-29T21:47:25.092Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 671 | 2024-05-29T21:47:25.110Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 672 | 2024-05-29T21:47:25.110Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 673 | 2024-05-29T21:47:25.129Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 674 | 2024-05-29T21:47:25.129Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 675 | 2024-05-29T21:47:25.184Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 676 | 2024-05-29T21:47:25.184Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 677 | 2024-05-29T21:47:25.239Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 678 | 2024-05-29T21:47:25.239Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 679 | 2024-05-29T21:47:25.294Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 680 | 2024-05-29T21:47:25.295Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 681 | 2024-05-29T21:47:25.346Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 682 | 2024-05-29T21:47:25.346Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 683 | 2024-05-29T21:47:25.398Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 684 | 2024-05-29T21:47:25.398Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 685 | 2024-05-29T21:47:25.398Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 686 | 2024-05-29T21:47:25.398Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 687 | 2024-05-29T21:47:25.431Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 688 | 2024-05-29T21:47:25.431Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 689 | 2024-05-29T21:47:25.431Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 690 | 2024-05-29T21:47:25.431Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 691 | 2024-05-29T21:47:25.431Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 692 | 2024-05-29T21:47:25.431Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 359384
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = e06d5871-2eeb-4bce-8cb7-d8132c14a8fc
response_code = 500
uri = /omicron-zones
|
| 693 | 2024-05-29T21:47:25.431Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e06d5871-2eeb-4bce-8cb7-d8132c14a8fc", "content-length": "124", "date": "Wed, 29 May 2024 21:47:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e06d5871-2eeb-4bce-8cb7-d8132c14a8fc" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 113.286797ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 694 | 2024-05-29T21:47:25.546Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 695 | 2024-05-29T21:47:25.546Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 696 | 2024-05-29T21:47:25.546Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 697 | 2024-05-29T21:47:25.564Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 698 | 2024-05-29T21:47:25.564Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 699 | 2024-05-29T21:47:25.582Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 700 | 2024-05-29T21:47:25.582Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 701 | 2024-05-29T21:47:25.600Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 702 | 2024-05-29T21:47:25.600Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 703 | 2024-05-29T21:47:25.618Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 704 | 2024-05-29T21:47:25.618Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 705 | 2024-05-29T21:47:25.636Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 706 | 2024-05-29T21:47:25.636Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 707 | 2024-05-29T21:47:25.654Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 708 | 2024-05-29T21:47:25.655Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 709 | 2024-05-29T21:47:25.672Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 710 | 2024-05-29T21:47:25.673Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 711 | 2024-05-29T21:47:25.690Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 712 | 2024-05-29T21:47:25.690Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 713 | 2024-05-29T21:47:25.690Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 714 | 2024-05-29T21:47:25.690Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 715 | 2024-05-29T21:47:25.722Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 716 | 2024-05-29T21:47:25.722Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 717 | 2024-05-29T21:47:25.722Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 718 | 2024-05-29T21:47:25.722Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 719 | 2024-05-29T21:47:25.722Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 720 | 2024-05-29T21:47:25.722Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 176643
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = 38cc9bb3-622f-4a6f-a8a4-1dcac8918b1c
response_code = 500
uri = /omicron-zones
|
| 721 | 2024-05-29T21:47:25.723Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "38cc9bb3-622f-4a6f-a8a4-1dcac8918b1c", "content-length": "124", "date": "Wed, 29 May 2024 21:47:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "38cc9bb3-622f-4a6f-a8a4-1dcac8918b1c" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 105.080144ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 722 | 2024-05-29T21:47:26.004Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 723 | 2024-05-29T21:47:26.004Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 724 | 2024-05-29T21:47:26.004Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 725 | 2024-05-29T21:47:26.023Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 726 | 2024-05-29T21:47:26.023Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 727 | 2024-05-29T21:47:26.040Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 728 | 2024-05-29T21:47:26.040Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 729 | 2024-05-29T21:47:26.058Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 730 | 2024-05-29T21:47:26.058Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 731 | 2024-05-29T21:47:26.075Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 732 | 2024-05-29T21:47:26.075Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 733 | 2024-05-29T21:47:26.094Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 734 | 2024-05-29T21:47:26.094Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 735 | 2024-05-29T21:47:26.112Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 736 | 2024-05-29T21:47:26.112Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 737 | 2024-05-29T21:47:26.130Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 738 | 2024-05-29T21:47:26.130Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 739 | 2024-05-29T21:47:26.147Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 740 | 2024-05-29T21:47:26.147Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 741 | 2024-05-29T21:47:26.147Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 742 | 2024-05-29T21:47:26.147Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 743 | 2024-05-29T21:47:26.179Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 744 | 2024-05-29T21:47:26.179Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 745 | 2024-05-29T21:47:26.179Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 746 | 2024-05-29T21:47:26.179Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 747 | 2024-05-29T21:47:26.179Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 748 | 2024-05-29T21:47:26.179Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 175270
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = 7b82ee99-e937-4328-a71d-db34ff674a0b
response_code = 500
uri = /omicron-zones
|
| 749 | 2024-05-29T21:47:26.179Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "7b82ee99-e937-4328-a71d-db34ff674a0b", "content-length": "124", "date": "Wed, 29 May 2024 21:47:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "7b82ee99-e937-4328-a71d-db34ff674a0b" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 175.191456ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 750 | 2024-05-29T21:47:26.356Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 751 | 2024-05-29T21:47:26.356Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 752 | 2024-05-29T21:47:26.356Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 753 | 2024-05-29T21:47:26.374Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 754 | 2024-05-29T21:47:26.374Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 755 | 2024-05-29T21:47:26.391Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 756 | 2024-05-29T21:47:26.391Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 757 | 2024-05-29T21:47:26.409Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 758 | 2024-05-29T21:47:26.409Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 759 | 2024-05-29T21:47:26.427Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 760 | 2024-05-29T21:47:26.427Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 761 | 2024-05-29T21:47:26.445Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 762 | 2024-05-29T21:47:26.445Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 763 | 2024-05-29T21:47:26.462Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 764 | 2024-05-29T21:47:26.462Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 765 | 2024-05-29T21:47:26.480Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 766 | 2024-05-29T21:47:26.480Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 767 | 2024-05-29T21:47:26.497Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 768 | 2024-05-29T21:47:26.497Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 769 | 2024-05-29T21:47:26.497Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 770 | 2024-05-29T21:47:26.498Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 771 | 2024-05-29T21:47:26.529Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 772 | 2024-05-29T21:47:26.529Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 773 | 2024-05-29T21:47:26.529Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 774 | 2024-05-29T21:47:26.529Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 775 | 2024-05-29T21:47:26.529Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 776 | 2024-05-29T21:47:26.529Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 172792
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = 65ff1e68-4c32-4515-8969-e38d7e0a2f5d
response_code = 500
uri = /omicron-zones
|
| 777 | 2024-05-29T21:47:26.529Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "65ff1e68-4c32-4515-8969-e38d7e0a2f5d", "content-length": "124", "date": "Wed, 29 May 2024 21:47:26 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "65ff1e68-4c32-4515-8969-e38d7e0a2f5d" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 212.379973ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 778 | 2024-05-29T21:47:26.743Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 779 | 2024-05-29T21:47:26.744Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 780 | 2024-05-29T21:47:26.744Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 781 | 2024-05-29T21:47:26.761Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 782 | 2024-05-29T21:47:26.761Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 783 | 2024-05-29T21:47:26.778Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 784 | 2024-05-29T21:47:26.778Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 785 | 2024-05-29T21:47:26.795Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 786 | 2024-05-29T21:47:26.795Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 787 | 2024-05-29T21:47:26.811Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 788 | 2024-05-29T21:47:26.811Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 789 | 2024-05-29T21:47:26.828Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 790 | 2024-05-29T21:47:26.828Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 791 | 2024-05-29T21:47:26.845Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 792 | 2024-05-29T21:47:26.845Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 793 | 2024-05-29T21:47:26.863Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 794 | 2024-05-29T21:47:26.863Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 795 | 2024-05-29T21:47:26.880Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 796 | 2024-05-29T21:47:26.880Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 797 | 2024-05-29T21:47:26.880Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 798 | 2024-05-29T21:47:26.880Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 799 | 2024-05-29T21:47:26.911Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 800 | 2024-05-29T21:47:26.911Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 801 | 2024-05-29T21:47:26.911Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 802 | 2024-05-29T21:47:26.911Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 803 | 2024-05-29T21:47:26.911Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 804 | 2024-05-29T21:47:26.911Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 167463
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = 7cd38bea-b456-46cd-8526-df26939f6af9
response_code = 500
uri = /omicron-zones
|
| 805 | 2024-05-29T21:47:26.911Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "7cd38bea-b456-46cd-8526-df26939f6af9", "content-length": "124", "date": "Wed, 29 May 2024 21:47:26 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "7cd38bea-b456-46cd-8526-df26939f6af9" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 197.780628ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 806 | 2024-05-29T21:47:27.109Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 807 | 2024-05-29T21:47:27.110Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 808 | 2024-05-29T21:47:27.110Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 809 | 2024-05-29T21:47:27.131Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 810 | 2024-05-29T21:47:27.131Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 811 | 2024-05-29T21:47:27.149Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 812 | 2024-05-29T21:47:27.150Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 813 | 2024-05-29T21:47:27.168Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 814 | 2024-05-29T21:47:27.168Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 815 | 2024-05-29T21:47:27.187Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 816 | 2024-05-29T21:47:27.187Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 817 | 2024-05-29T21:47:27.205Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 818 | 2024-05-29T21:47:27.205Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 819 | 2024-05-29T21:47:27.223Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 820 | 2024-05-29T21:47:27.223Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 821 | 2024-05-29T21:47:27.242Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 822 | 2024-05-29T21:47:27.242Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 823 | 2024-05-29T21:47:27.260Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 824 | 2024-05-29T21:47:27.260Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 825 | 2024-05-29T21:47:27.260Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 826 | 2024-05-29T21:47:27.260Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 827 | 2024-05-29T21:47:27.293Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 828 | 2024-05-29T21:47:27.293Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 829 | 2024-05-29T21:47:27.294Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 830 | 2024-05-29T21:47:27.294Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 831 | 2024-05-29T21:47:27.294Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 832 | 2024-05-29T21:47:27.294Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 184091
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = c08ddd2f-852b-4b90-89fb-adebd25e53a9
response_code = 500
uri = /omicron-zones
|
| 833 | 2024-05-29T21:47:27.294Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "c08ddd2f-852b-4b90-89fb-adebd25e53a9", "content-length": "124", "date": "Wed, 29 May 2024 21:47:26 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "c08ddd2f-852b-4b90-89fb-adebd25e53a9" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 141.028109ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 834 | 2024-05-29T21:47:27.436Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 835 | 2024-05-29T21:47:27.437Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 836 | 2024-05-29T21:47:27.437Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 837 | 2024-05-29T21:47:27.456Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 838 | 2024-05-29T21:47:27.456Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 839 | 2024-05-29T21:47:27.473Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 840 | 2024-05-29T21:47:27.474Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 841 | 2024-05-29T21:47:27.491Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 842 | 2024-05-29T21:47:27.491Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 843 | 2024-05-29T21:47:27.509Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 844 | 2024-05-29T21:47:27.509Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 845 | 2024-05-29T21:47:27.527Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 846 | 2024-05-29T21:47:27.527Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 847 | 2024-05-29T21:47:27.545Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 848 | 2024-05-29T21:47:27.545Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 849 | 2024-05-29T21:47:27.564Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 850 | 2024-05-29T21:47:27.564Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 851 | 2024-05-29T21:47:27.582Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 852 | 2024-05-29T21:47:27.582Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 853 | 2024-05-29T21:47:27.583Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 854 | 2024-05-29T21:47:27.583Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 855 | 2024-05-29T21:47:27.616Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 856 | 2024-05-29T21:47:27.616Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 857 | 2024-05-29T21:47:27.616Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 858 | 2024-05-29T21:47:27.616Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 859 | 2024-05-29T21:47:27.616Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 860 | 2024-05-29T21:47:27.616Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 179395
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = be89319e-aea9-4c12-a9f0-4ac773d3ded7
response_code = 500
uri = /omicron-zones
|
| 861 | 2024-05-29T21:47:27.616Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "be89319e-aea9-4c12-a9f0-4ac773d3ded7", "content-length": "124", "date": "Wed, 29 May 2024 21:47:27 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "be89319e-aea9-4c12-a9f0-4ac773d3ded7" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 374.082058ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 862 | 2024-05-29T21:47:27.991Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 863 | 2024-05-29T21:47:27.991Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 864 | 2024-05-29T21:47:27.991Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 865 | 2024-05-29T21:47:28.012Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 866 | 2024-05-29T21:47:28.012Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 867 | 2024-05-29T21:47:28.031Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 868 | 2024-05-29T21:47:28.031Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 869 | 2024-05-29T21:47:28.051Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 870 | 2024-05-29T21:47:28.051Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 871 | 2024-05-29T21:47:28.069Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 872 | 2024-05-29T21:47:28.069Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 873 | 2024-05-29T21:47:28.087Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 874 | 2024-05-29T21:47:28.087Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 875 | 2024-05-29T21:47:28.106Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 876 | 2024-05-29T21:47:28.106Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 877 | 2024-05-29T21:47:28.124Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 878 | 2024-05-29T21:47:28.124Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 879 | 2024-05-29T21:47:28.142Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 880 | 2024-05-29T21:47:28.143Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 881 | 2024-05-29T21:47:28.143Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 882 | 2024-05-29T21:47:28.143Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 883 | 2024-05-29T21:47:28.175Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 884 | 2024-05-29T21:47:28.175Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 885 | 2024-05-29T21:47:28.175Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 886 | 2024-05-29T21:47:28.175Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 887 | 2024-05-29T21:47:28.175Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 888 | 2024-05-29T21:47:28.175Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 183917
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = 4877be18-df25-4fbe-9b5e-1b7e7c6b1aa2
response_code = 500
uri = /omicron-zones
|
| 889 | 2024-05-29T21:47:28.175Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "4877be18-df25-4fbe-9b5e-1b7e7c6b1aa2", "content-length": "124", "date": "Wed, 29 May 2024 21:47:27 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "4877be18-df25-4fbe-9b5e-1b7e7c6b1aa2" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 422.208795ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 890 | 2024-05-29T21:47:28.598Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 891 | 2024-05-29T21:47:28.599Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 892 | 2024-05-29T21:47:28.599Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 893 | 2024-05-29T21:47:28.618Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 894 | 2024-05-29T21:47:28.618Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 895 | 2024-05-29T21:47:28.635Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 896 | 2024-05-29T21:47:28.635Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 897 | 2024-05-29T21:47:28.654Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 898 | 2024-05-29T21:47:28.654Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 899 | 2024-05-29T21:47:28.672Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 900 | 2024-05-29T21:47:28.672Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 901 | 2024-05-29T21:47:28.690Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 902 | 2024-05-29T21:47:28.690Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 903 | 2024-05-29T21:47:28.708Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 904 | 2024-05-29T21:47:28.708Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 905 | 2024-05-29T21:47:28.726Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 906 | 2024-05-29T21:47:28.726Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 907 | 2024-05-29T21:47:28.744Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 908 | 2024-05-29T21:47:28.744Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 909 | 2024-05-29T21:47:28.744Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 910 | 2024-05-29T21:47:28.744Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 911 | 2024-05-29T21:47:28.775Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 912 | 2024-05-29T21:47:28.775Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 913 | 2024-05-29T21:47:28.775Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 914 | 2024-05-29T21:47:28.775Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 915 | 2024-05-29T21:47:28.775Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 916 | 2024-05-29T21:47:28.775Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 176775
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = e61b2b07-9dcd-424e-8bed-2a07d768e12c
response_code = 500
uri = /omicron-zones
|
| 917 | 2024-05-29T21:47:28.775Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e61b2b07-9dcd-424e-8bed-2a07d768e12c", "content-length": "124", "date": "Wed, 29 May 2024 21:47:27 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e61b2b07-9dcd-424e-8bed-2a07d768e12c" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 547.965627ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 918 | 2024-05-29T21:47:29.325Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 919 | 2024-05-29T21:47:29.326Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 920 | 2024-05-29T21:47:29.326Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 921 | 2024-05-29T21:47:29.345Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 922 | 2024-05-29T21:47:29.345Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 923 | 2024-05-29T21:47:29.363Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 924 | 2024-05-29T21:47:29.363Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 925 | 2024-05-29T21:47:29.381Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 926 | 2024-05-29T21:47:29.381Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 927 | 2024-05-29T21:47:29.400Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 928 | 2024-05-29T21:47:29.400Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 929 | 2024-05-29T21:47:29.418Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 930 | 2024-05-29T21:47:29.418Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 931 | 2024-05-29T21:47:29.436Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 932 | 2024-05-29T21:47:29.436Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 933 | 2024-05-29T21:47:29.455Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 934 | 2024-05-29T21:47:29.455Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 935 | 2024-05-29T21:47:29.472Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 936 | 2024-05-29T21:47:29.472Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 937 | 2024-05-29T21:47:29.472Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 938 | 2024-05-29T21:47:29.473Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 939 | 2024-05-29T21:47:29.504Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 940 | 2024-05-29T21:47:29.504Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 941 | 2024-05-29T21:47:29.504Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 942 | 2024-05-29T21:47:29.504Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 943 | 2024-05-29T21:47:29.504Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 944 | 2024-05-29T21:47:29.505Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 179580
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = 5494ba55-d0ed-4f86-a4bd-efd6167c45a8
response_code = 500
uri = /omicron-zones
|
| 945 | 2024-05-29T21:47:29.505Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "5494ba55-d0ed-4f86-a4bd-efd6167c45a8", "content-length": "124", "date": "Wed, 29 May 2024 21:47:29 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "5494ba55-d0ed-4f86-a4bd-efd6167c45a8" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 327.310334ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 946 | 2024-05-29T21:47:29.833Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 947 | 2024-05-29T21:47:29.833Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 948 | 2024-05-29T21:47:29.833Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 949 | 2024-05-29T21:47:29.851Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 950 | 2024-05-29T21:47:29.851Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 951 | 2024-05-29T21:47:29.869Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 952 | 2024-05-29T21:47:29.869Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 953 | 2024-05-29T21:47:29.886Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 954 | 2024-05-29T21:47:29.886Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 955 | 2024-05-29T21:47:29.904Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 956 | 2024-05-29T21:47:29.904Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 957 | 2024-05-29T21:47:29.922Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 958 | 2024-05-29T21:47:29.922Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 959 | 2024-05-29T21:47:29.940Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 960 | 2024-05-29T21:47:29.941Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 961 | 2024-05-29T21:47:29.958Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 962 | 2024-05-29T21:47:29.958Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 963 | 2024-05-29T21:47:29.976Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 964 | 2024-05-29T21:47:29.976Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 965 | 2024-05-29T21:47:29.976Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 966 | 2024-05-29T21:47:29.976Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 967 | 2024-05-29T21:47:30.008Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 968 | 2024-05-29T21:47:30.008Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 969 | 2024-05-29T21:47:30.008Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 970 | 2024-05-29T21:47:30.008Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 971 | 2024-05-29T21:47:30.008Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 972 | 2024-05-29T21:47:30.008Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 174622
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = aafd9283-6d30-4d06-9806-5abcde7e4942
response_code = 500
uri = /omicron-zones
|
| 973 | 2024-05-29T21:47:30.008Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "aafd9283-6d30-4d06-9806-5abcde7e4942", "content-length": "124", "date": "Wed, 29 May 2024 21:47:29 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "aafd9283-6d30-4d06-9806-5abcde7e4942" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 879.73787ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 974 | 2024-05-29T21:47:30.889Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 975 | 2024-05-29T21:47:30.889Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 976 | 2024-05-29T21:47:30.889Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 977 | 2024-05-29T21:47:30.909Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 978 | 2024-05-29T21:47:30.909Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 979 | 2024-05-29T21:47:30.927Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 980 | 2024-05-29T21:47:30.927Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 981 | 2024-05-29T21:47:30.945Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 982 | 2024-05-29T21:47:30.945Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 983 | 2024-05-29T21:47:30.964Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 984 | 2024-05-29T21:47:30.964Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 985 | 2024-05-29T21:47:30.981Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 986 | 2024-05-29T21:47:30.982Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 987 | 2024-05-29T21:47:30.999Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 988 | 2024-05-29T21:47:30.999Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 989 | 2024-05-29T21:47:31.019Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 990 | 2024-05-29T21:47:31.019Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 991 | 2024-05-29T21:47:31.037Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 992 | 2024-05-29T21:47:31.037Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 993 | 2024-05-29T21:47:31.037Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 994 | 2024-05-29T21:47:31.037Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 995 | 2024-05-29T21:47:31.068Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 996 | 2024-05-29T21:47:31.068Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 997 | 2024-05-29T21:47:31.068Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 998 | 2024-05-29T21:47:31.068Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 999 | 2024-05-29T21:47:31.068Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1000 | 2024-05-29T21:47:31.069Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 179794
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = af7ffdf4-b20b-44f5-baa5-0d85b99f709d
response_code = 500
uri = /omicron-zones
|
| 1001 | 2024-05-29T21:47:31.069Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "af7ffdf4-b20b-44f5-baa5-0d85b99f709d", "content-length": "124", "date": "Wed, 29 May 2024 21:47:31 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "af7ffdf4-b20b-44f5-baa5-0d85b99f709d" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 644.483817ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1002 | 2024-05-29T21:47:31.714Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1003 | 2024-05-29T21:47:31.714Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1004 | 2024-05-29T21:47:31.714Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1005 | 2024-05-29T21:47:31.733Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1006 | 2024-05-29T21:47:31.733Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1007 | 2024-05-29T21:47:31.752Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1008 | 2024-05-29T21:47:31.752Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1009 | 2024-05-29T21:47:31.770Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1010 | 2024-05-29T21:47:31.770Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1011 | 2024-05-29T21:47:31.787Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1012 | 2024-05-29T21:47:31.787Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1013 | 2024-05-29T21:47:31.805Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1014 | 2024-05-29T21:47:31.805Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1015 | 2024-05-29T21:47:31.823Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1016 | 2024-05-29T21:47:31.823Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1017 | 2024-05-29T21:47:31.842Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1018 | 2024-05-29T21:47:31.842Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1019 | 2024-05-29T21:47:31.860Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1020 | 2024-05-29T21:47:31.860Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1021 | 2024-05-29T21:47:31.860Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1022 | 2024-05-29T21:47:31.861Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1023 | 2024-05-29T21:47:31.893Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1024 | 2024-05-29T21:47:31.893Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1025 | 2024-05-29T21:47:31.893Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1026 | 2024-05-29T21:47:31.893Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1027 | 2024-05-29T21:47:31.893Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1028 | 2024-05-29T21:47:31.893Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 178633
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = b2251b1c-234e-40c5-8e08-116b3b8244fb
response_code = 500
uri = /omicron-zones
|
| 1029 | 2024-05-29T21:47:31.893Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b2251b1c-234e-40c5-8e08-116b3b8244fb", "content-length": "124", "date": "Wed, 29 May 2024 21:47:31 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b2251b1c-234e-40c5-8e08-116b3b8244fb" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 1.040193964s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1030 | 2024-05-29T21:47:32.934Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1031 | 2024-05-29T21:47:32.935Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1032 | 2024-05-29T21:47:32.935Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1033 | 2024-05-29T21:47:32.954Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1034 | 2024-05-29T21:47:32.954Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1035 | 2024-05-29T21:47:32.972Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1036 | 2024-05-29T21:47:32.972Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1037 | 2024-05-29T21:47:32.990Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1038 | 2024-05-29T21:47:32.990Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1039 | 2024-05-29T21:47:33.009Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1040 | 2024-05-29T21:47:33.009Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1041 | 2024-05-29T21:47:33.027Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1042 | 2024-05-29T21:47:33.027Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1043 | 2024-05-29T21:47:33.046Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1044 | 2024-05-29T21:47:33.046Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1045 | 2024-05-29T21:47:33.064Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1046 | 2024-05-29T21:47:33.064Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1047 | 2024-05-29T21:47:33.082Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1048 | 2024-05-29T21:47:33.082Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1049 | 2024-05-29T21:47:33.082Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1050 | 2024-05-29T21:47:33.082Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1051 | 2024-05-29T21:47:33.115Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1052 | 2024-05-29T21:47:33.116Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1053 | 2024-05-29T21:47:33.116Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1054 | 2024-05-29T21:47:33.116Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1055 | 2024-05-29T21:47:33.116Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1056 | 2024-05-29T21:47:33.116Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 181223
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = ff8f8d98-7316-47ad-836d-0fc81f41207a
response_code = 500
uri = /omicron-zones
|
| 1057 | 2024-05-29T21:47:33.116Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "ff8f8d98-7316-47ad-836d-0fc81f41207a", "content-length": "124", "date": "Wed, 29 May 2024 21:47:32 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "ff8f8d98-7316-47ad-836d-0fc81f41207a" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 723.378896ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1058 | 2024-05-29T21:47:33.840Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1059 | 2024-05-29T21:47:33.841Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1060 | 2024-05-29T21:47:33.841Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1061 | 2024-05-29T21:47:33.861Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1062 | 2024-05-29T21:47:33.861Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1063 | 2024-05-29T21:47:33.879Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1064 | 2024-05-29T21:47:33.879Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1065 | 2024-05-29T21:47:33.897Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1066 | 2024-05-29T21:47:33.897Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1067 | 2024-05-29T21:47:33.917Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1068 | 2024-05-29T21:47:33.917Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1069 | 2024-05-29T21:47:33.934Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1070 | 2024-05-29T21:47:33.934Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1071 | 2024-05-29T21:47:33.952Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1072 | 2024-05-29T21:47:33.952Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1073 | 2024-05-29T21:47:33.970Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1074 | 2024-05-29T21:47:33.970Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1075 | 2024-05-29T21:47:33.988Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1076 | 2024-05-29T21:47:33.988Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1077 | 2024-05-29T21:47:33.988Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1078 | 2024-05-29T21:47:33.988Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1079 | 2024-05-29T21:47:34.020Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1080 | 2024-05-29T21:47:34.020Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1081 | 2024-05-29T21:47:34.020Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1082 | 2024-05-29T21:47:34.020Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1083 | 2024-05-29T21:47:34.020Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1084 | 2024-05-29T21:47:34.020Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 179664
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = e3477172-0956-44c2-99a6-b03e0fad1308
response_code = 500
uri = /omicron-zones
|
| 1085 | 2024-05-29T21:47:34.020Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e3477172-0956-44c2-99a6-b03e0fad1308", "content-length": "124", "date": "Wed, 29 May 2024 21:47:34 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e3477172-0956-44c2-99a6-b03e0fad1308" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 1.776763671s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1086 | 2024-05-29T21:47:35.798Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1087 | 2024-05-29T21:47:35.798Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1088 | 2024-05-29T21:47:35.798Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1089 | 2024-05-29T21:47:35.818Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1090 | 2024-05-29T21:47:35.818Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1091 | 2024-05-29T21:47:35.837Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1092 | 2024-05-29T21:47:35.837Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1093 | 2024-05-29T21:47:35.855Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1094 | 2024-05-29T21:47:35.855Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1095 | 2024-05-29T21:47:35.873Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1096 | 2024-05-29T21:47:35.873Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1097 | 2024-05-29T21:47:35.891Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1098 | 2024-05-29T21:47:35.891Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1099 | 2024-05-29T21:47:35.911Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1100 | 2024-05-29T21:47:35.911Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1101 | 2024-05-29T21:47:35.929Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1102 | 2024-05-29T21:47:35.929Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1103 | 2024-05-29T21:47:35.947Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1104 | 2024-05-29T21:47:35.947Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1105 | 2024-05-29T21:47:35.947Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1106 | 2024-05-29T21:47:35.947Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1107 | 2024-05-29T21:47:35.981Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1108 | 2024-05-29T21:47:35.981Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1109 | 2024-05-29T21:47:35.981Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1110 | 2024-05-29T21:47:35.981Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1111 | 2024-05-29T21:47:35.981Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1112 | 2024-05-29T21:47:35.981Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 182990
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = d2f6f184-faf7-4bea-8f47-d1219f5f8edb
response_code = 500
uri = /omicron-zones
|
| 1113 | 2024-05-29T21:47:35.981Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d2f6f184-faf7-4bea-8f47-d1219f5f8edb", "content-length": "124", "date": "Wed, 29 May 2024 21:47:35 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d2f6f184-faf7-4bea-8f47-d1219f5f8edb" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 1.684032411s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1114 | 2024-05-29T21:47:37.666Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1115 | 2024-05-29T21:47:37.666Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1116 | 2024-05-29T21:47:37.666Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1117 | 2024-05-29T21:47:37.687Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1118 | 2024-05-29T21:47:37.687Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1119 | 2024-05-29T21:47:37.705Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1120 | 2024-05-29T21:47:37.705Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1121 | 2024-05-29T21:47:37.723Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1122 | 2024-05-29T21:47:37.723Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1123 | 2024-05-29T21:47:37.743Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1124 | 2024-05-29T21:47:37.743Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1125 | 2024-05-29T21:47:37.764Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1126 | 2024-05-29T21:47:37.764Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1127 | 2024-05-29T21:47:37.782Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1128 | 2024-05-29T21:47:37.782Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1129 | 2024-05-29T21:47:37.801Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1130 | 2024-05-29T21:47:37.801Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1131 | 2024-05-29T21:47:37.819Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1132 | 2024-05-29T21:47:37.819Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1133 | 2024-05-29T21:47:37.819Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1134 | 2024-05-29T21:47:37.819Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1135 | 2024-05-29T21:47:37.851Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1136 | 2024-05-29T21:47:37.851Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1137 | 2024-05-29T21:47:37.851Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1138 | 2024-05-29T21:47:37.851Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1139 | 2024-05-29T21:47:37.851Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1140 | 2024-05-29T21:47:37.851Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 185117
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = 0cee7879-aed7-453e-b7c6-a57b38fae087
response_code = 500
uri = /omicron-zones
|
| 1141 | 2024-05-29T21:47:37.851Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "0cee7879-aed7-453e-b7c6-a57b38fae087", "content-length": "124", "date": "Wed, 29 May 2024 21:47:37 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "0cee7879-aed7-453e-b7c6-a57b38fae087" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 2.039214865s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1142 | 2024-05-29T21:47:39.315Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 290.570094ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:417
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 1143 | 2024-05-29T21:47:39.892Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1144 | 2024-05-29T21:47:39.892Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1145 | 2024-05-29T21:47:39.892Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1146 | 2024-05-29T21:47:39.912Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1147 | 2024-05-29T21:47:39.912Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1148 | 2024-05-29T21:47:39.929Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1149 | 2024-05-29T21:47:39.929Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1150 | 2024-05-29T21:47:39.947Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1151 | 2024-05-29T21:47:39.947Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1152 | 2024-05-29T21:47:39.966Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1153 | 2024-05-29T21:47:39.966Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1154 | 2024-05-29T21:47:39.985Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1155 | 2024-05-29T21:47:39.985Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1156 | 2024-05-29T21:47:40.004Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1157 | 2024-05-29T21:47:40.004Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1158 | 2024-05-29T21:47:40.023Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1159 | 2024-05-29T21:47:40.023Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1160 | 2024-05-29T21:47:40.041Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1161 | 2024-05-29T21:47:40.041Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1162 | 2024-05-29T21:47:40.041Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1163 | 2024-05-29T21:47:40.041Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1164 | 2024-05-29T21:47:40.073Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1165 | 2024-05-29T21:47:40.073Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1166 | 2024-05-29T21:47:40.073Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1167 | 2024-05-29T21:47:40.073Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1168 | 2024-05-29T21:47:40.073Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1169 | 2024-05-29T21:47:40.073Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 180624
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = 8f138a61-20ed-4750-ac47-2d89eb45ec33
response_code = 500
uri = /omicron-zones
|
| 1170 | 2024-05-29T21:47:40.073Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "8f138a61-20ed-4750-ac47-2d89eb45ec33", "content-length": "124", "date": "Wed, 29 May 2024 21:47:39 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "8f138a61-20ed-4750-ac47-2d89eb45ec33" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 3.278090756s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1171 | 2024-05-29T21:47:40.573Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:41532
|
| 1172 | 2024-05-29T21:47:40.573Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 53
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:41532
req_id = 3662d255-3ca2-4b6f-a059-f30fb44dfd96
response_code = 200
uri = /baseboard
|
| 1173 | 2024-05-29T21:47:43.352Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1174 | 2024-05-29T21:47:43.353Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1175 | 2024-05-29T21:47:43.353Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1176 | 2024-05-29T21:47:43.375Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1177 | 2024-05-29T21:47:43.375Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1178 | 2024-05-29T21:47:43.394Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1179 | 2024-05-29T21:47:43.394Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1180 | 2024-05-29T21:47:43.412Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1181 | 2024-05-29T21:47:43.412Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1182 | 2024-05-29T21:47:43.429Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1183 | 2024-05-29T21:47:43.429Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1184 | 2024-05-29T21:47:43.447Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1185 | 2024-05-29T21:47:43.447Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1186 | 2024-05-29T21:47:43.465Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1187 | 2024-05-29T21:47:43.465Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1188 | 2024-05-29T21:47:43.483Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1189 | 2024-05-29T21:47:43.483Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1190 | 2024-05-29T21:47:43.500Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1191 | 2024-05-29T21:47:43.500Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1192 | 2024-05-29T21:47:43.500Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1193 | 2024-05-29T21:47:43.501Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1194 | 2024-05-29T21:47:43.531Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1195 | 2024-05-29T21:47:43.531Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1196 | 2024-05-29T21:47:43.531Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1197 | 2024-05-29T21:47:43.531Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1198 | 2024-05-29T21:47:43.531Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1199 | 2024-05-29T21:47:43.531Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 178954
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = e813c706-ec04-4f47-a38e-bf3c9453d6cf
response_code = 500
uri = /omicron-zones
|
| 1200 | 2024-05-29T21:47:43.531Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e813c706-ec04-4f47-a38e-bf3c9453d6cf", "content-length": "124", "date": "Wed, 29 May 2024 21:47:43 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e813c706-ec04-4f47-a38e-bf3c9453d6cf" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 2.514669322s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1201 | 2024-05-29T21:47:46.047Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1202 | 2024-05-29T21:47:46.047Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1203 | 2024-05-29T21:47:46.047Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1204 | 2024-05-29T21:47:46.068Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1205 | 2024-05-29T21:47:46.068Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1206 | 2024-05-29T21:47:46.087Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1207 | 2024-05-29T21:47:46.087Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1208 | 2024-05-29T21:47:46.105Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1209 | 2024-05-29T21:47:46.105Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1210 | 2024-05-29T21:47:46.124Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1211 | 2024-05-29T21:47:46.124Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1212 | 2024-05-29T21:47:46.142Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1213 | 2024-05-29T21:47:46.142Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1214 | 2024-05-29T21:47:46.160Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1215 | 2024-05-29T21:47:46.160Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1216 | 2024-05-29T21:47:46.179Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1217 | 2024-05-29T21:47:46.179Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1218 | 2024-05-29T21:47:46.198Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1219 | 2024-05-29T21:47:46.198Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1220 | 2024-05-29T21:47:46.198Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1221 | 2024-05-29T21:47:46.198Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1222 | 2024-05-29T21:47:46.231Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1223 | 2024-05-29T21:47:46.231Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1224 | 2024-05-29T21:47:46.231Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1225 | 2024-05-29T21:47:46.231Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1226 | 2024-05-29T21:47:46.231Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1227 | 2024-05-29T21:47:46.231Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 184102
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = e1461f15-f166-4a5d-a9a8-d0eb106eb041
response_code = 500
uri = /omicron-zones
|
| 1228 | 2024-05-29T21:47:46.231Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e1461f15-f166-4a5d-a9a8-d0eb106eb041", "content-length": "124", "date": "Wed, 29 May 2024 21:47:46 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e1461f15-f166-4a5d-a9a8-d0eb106eb041" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 2.327853055s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1229 | 2024-05-29T21:47:48.561Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1230 | 2024-05-29T21:47:48.561Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1231 | 2024-05-29T21:47:48.561Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1232 | 2024-05-29T21:47:48.581Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1233 | 2024-05-29T21:47:48.581Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1234 | 2024-05-29T21:47:48.600Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1235 | 2024-05-29T21:47:48.600Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1236 | 2024-05-29T21:47:48.619Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1237 | 2024-05-29T21:47:48.619Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1238 | 2024-05-29T21:47:48.637Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1239 | 2024-05-29T21:47:48.637Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1240 | 2024-05-29T21:47:48.655Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1241 | 2024-05-29T21:47:48.655Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1242 | 2024-05-29T21:47:48.673Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1243 | 2024-05-29T21:47:48.673Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1244 | 2024-05-29T21:47:48.692Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1245 | 2024-05-29T21:47:48.692Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1246 | 2024-05-29T21:47:48.710Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1247 | 2024-05-29T21:47:48.710Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1248 | 2024-05-29T21:47:48.710Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1249 | 2024-05-29T21:47:48.711Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1250 | 2024-05-29T21:47:48.743Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1251 | 2024-05-29T21:47:48.743Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1252 | 2024-05-29T21:47:48.743Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1253 | 2024-05-29T21:47:48.743Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1254 | 2024-05-29T21:47:48.743Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1255 | 2024-05-29T21:47:48.743Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 181635
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = 500e9aec-a45e-4996-83d8-4f9dfea32942
response_code = 500
uri = /omicron-zones
|
| 1256 | 2024-05-29T21:47:48.744Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "500e9aec-a45e-4996-83d8-4f9dfea32942", "content-length": "124", "date": "Wed, 29 May 2024 21:47:48 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "500e9aec-a45e-4996-83d8-4f9dfea32942" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 2.360879845s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1257 | 2024-05-29T21:47:51.113Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1258 | 2024-05-29T21:47:51.113Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1259 | 2024-05-29T21:47:51.113Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1260 | 2024-05-29T21:47:51.133Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1261 | 2024-05-29T21:47:51.133Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1262 | 2024-05-29T21:47:51.153Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1263 | 2024-05-29T21:47:51.153Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1264 | 2024-05-29T21:47:51.172Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1265 | 2024-05-29T21:47:51.172Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1266 | 2024-05-29T21:47:51.189Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1267 | 2024-05-29T21:47:51.189Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1268 | 2024-05-29T21:47:51.208Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1269 | 2024-05-29T21:47:51.208Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1270 | 2024-05-29T21:47:51.225Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1271 | 2024-05-29T21:47:51.225Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1272 | 2024-05-29T21:47:51.243Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1273 | 2024-05-29T21:47:51.243Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1274 | 2024-05-29T21:47:51.262Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1275 | 2024-05-29T21:47:51.262Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1276 | 2024-05-29T21:47:51.262Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1277 | 2024-05-29T21:47:51.263Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1278 | 2024-05-29T21:47:51.293Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1279 | 2024-05-29T21:47:51.293Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1280 | 2024-05-29T21:47:51.293Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1281 | 2024-05-29T21:47:51.293Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1282 | 2024-05-29T21:47:51.293Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1283 | 2024-05-29T21:47:51.294Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 180263
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = 496b2f7a-8e3c-48a0-811a-86a9957c57c4
response_code = 500
uri = /omicron-zones
|
| 1284 | 2024-05-29T21:47:51.294Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "496b2f7a-8e3c-48a0-811a-86a9957c57c4", "content-length": "124", "date": "Wed, 29 May 2024 21:47:51 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "496b2f7a-8e3c-48a0-811a-86a9957c57c4" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 3.920021109s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1285 | 2024-05-29T21:47:54.659Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 770.826492ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:417
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 1286 | 2024-05-29T21:47:55.225Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1287 | 2024-05-29T21:47:55.225Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1288 | 2024-05-29T21:47:55.225Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1289 | 2024-05-29T21:47:55.245Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1290 | 2024-05-29T21:47:55.245Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1291 | 2024-05-29T21:47:55.264Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1292 | 2024-05-29T21:47:55.264Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1293 | 2024-05-29T21:47:55.282Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1294 | 2024-05-29T21:47:55.282Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1295 | 2024-05-29T21:47:55.300Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1296 | 2024-05-29T21:47:55.300Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1297 | 2024-05-29T21:47:55.318Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1298 | 2024-05-29T21:47:55.318Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1299 | 2024-05-29T21:47:55.336Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1300 | 2024-05-29T21:47:55.336Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1301 | 2024-05-29T21:47:55.353Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1302 | 2024-05-29T21:47:55.353Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1303 | 2024-05-29T21:47:55.371Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1304 | 2024-05-29T21:47:55.371Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1305 | 2024-05-29T21:47:55.371Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1306 | 2024-05-29T21:47:55.371Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1307 | 2024-05-29T21:47:55.403Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1308 | 2024-05-29T21:47:55.403Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1309 | 2024-05-29T21:47:55.403Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1310 | 2024-05-29T21:47:55.403Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1311 | 2024-05-29T21:47:55.403Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1312 | 2024-05-29T21:47:55.403Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 177386
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = bb79fa49-2f43-4356-a15d-17ba4b824af2
response_code = 500
uri = /omicron-zones
|
| 1313 | 2024-05-29T21:47:55.403Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "bb79fa49-2f43-4356-a15d-17ba4b824af2", "content-length": "124", "date": "Wed, 29 May 2024 21:47:55 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "bb79fa49-2f43-4356-a15d-17ba4b824af2" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 5.472356996s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1314 | 2024-05-29T21:48:00.891Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1315 | 2024-05-29T21:48:00.893Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1316 | 2024-05-29T21:48:00.893Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1317 | 2024-05-29T21:48:00.913Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1318 | 2024-05-29T21:48:00.913Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1319 | 2024-05-29T21:48:00.930Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1320 | 2024-05-29T21:48:00.930Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1321 | 2024-05-29T21:48:00.948Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1322 | 2024-05-29T21:48:00.948Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1323 | 2024-05-29T21:48:00.963Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1324 | 2024-05-29T21:48:00.963Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1325 | 2024-05-29T21:48:00.981Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1326 | 2024-05-29T21:48:00.981Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1327 | 2024-05-29T21:48:00.997Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1328 | 2024-05-29T21:48:00.997Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1329 | 2024-05-29T21:48:01.019Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1330 | 2024-05-29T21:48:01.019Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1331 | 2024-05-29T21:48:01.036Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1332 | 2024-05-29T21:48:01.036Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1333 | 2024-05-29T21:48:01.036Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1334 | 2024-05-29T21:48:01.036Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1335 | 2024-05-29T21:48:01.067Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1336 | 2024-05-29T21:48:01.067Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1337 | 2024-05-29T21:48:01.067Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1338 | 2024-05-29T21:48:01.067Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1339 | 2024-05-29T21:48:01.067Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1340 | 2024-05-29T21:48:01.067Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 175420
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = 4850fc90-cbad-4969-b915-5c7901ce83e4
response_code = 500
uri = /omicron-zones
|
| 1341 | 2024-05-29T21:48:01.068Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "4850fc90-cbad-4969-b915-5c7901ce83e4", "content-length": "124", "date": "Wed, 29 May 2024 21:48:01 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "4850fc90-cbad-4969-b915-5c7901ce83e4" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 7.448746006s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1342 | 2024-05-29T21:48:08.537Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1343 | 2024-05-29T21:48:08.537Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1344 | 2024-05-29T21:48:08.537Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1345 | 2024-05-29T21:48:08.558Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1346 | 2024-05-29T21:48:08.558Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1347 | 2024-05-29T21:48:08.578Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1348 | 2024-05-29T21:48:08.578Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1349 | 2024-05-29T21:48:08.596Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1350 | 2024-05-29T21:48:08.596Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1351 | 2024-05-29T21:48:08.616Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1352 | 2024-05-29T21:48:08.616Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1353 | 2024-05-29T21:48:08.634Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1354 | 2024-05-29T21:48:08.634Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1355 | 2024-05-29T21:48:08.652Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1356 | 2024-05-29T21:48:08.652Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1357 | 2024-05-29T21:48:08.670Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1358 | 2024-05-29T21:48:08.670Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1359 | 2024-05-29T21:48:08.688Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1360 | 2024-05-29T21:48:08.688Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1361 | 2024-05-29T21:48:08.688Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1362 | 2024-05-29T21:48:08.689Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1363 | 2024-05-29T21:48:08.721Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1364 | 2024-05-29T21:48:08.721Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1365 | 2024-05-29T21:48:08.721Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1366 | 2024-05-29T21:48:08.721Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1367 | 2024-05-29T21:48:08.721Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1368 | 2024-05-29T21:48:08.721Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 183420
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = 11b63411-4ac7-453f-aa03-ab683c6ac31b
response_code = 500
uri = /omicron-zones
|
| 1369 | 2024-05-29T21:48:08.721Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "11b63411-4ac7-453f-aa03-ab683c6ac31b", "content-length": "124", "date": "Wed, 29 May 2024 21:48:08 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "11b63411-4ac7-453f-aa03-ab683c6ac31b" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 10.949542614s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1370 | 2024-05-29T21:48:10.510Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 1.415104219s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:417
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 1371 | 2024-05-29T21:48:10.665Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:60197
|
| 1372 | 2024-05-29T21:48:10.665Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 34
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:60197
req_id = a8265dfd-1b23-46ff-984c-536d45cad767
response_code = 200
uri = /baseboard
|
| 1373 | 2024-05-29T21:48:19.702Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1374 | 2024-05-29T21:48:19.702Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1375 | 2024-05-29T21:48:19.702Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1376 | 2024-05-29T21:48:19.724Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1377 | 2024-05-29T21:48:19.724Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1378 | 2024-05-29T21:48:19.742Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1379 | 2024-05-29T21:48:19.742Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1380 | 2024-05-29T21:48:19.760Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1381 | 2024-05-29T21:48:19.760Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1382 | 2024-05-29T21:48:19.779Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1383 | 2024-05-29T21:48:19.779Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1384 | 2024-05-29T21:48:19.797Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1385 | 2024-05-29T21:48:19.797Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1386 | 2024-05-29T21:48:19.814Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1387 | 2024-05-29T21:48:19.814Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1388 | 2024-05-29T21:48:19.833Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1389 | 2024-05-29T21:48:19.833Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1390 | 2024-05-29T21:48:19.850Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1391 | 2024-05-29T21:48:19.850Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1392 | 2024-05-29T21:48:19.851Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1393 | 2024-05-29T21:48:19.851Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1394 | 2024-05-29T21:48:19.884Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1395 | 2024-05-29T21:48:19.884Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1396 | 2024-05-29T21:48:19.884Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1397 | 2024-05-29T21:48:19.884Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1398 | 2024-05-29T21:48:19.884Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1399 | 2024-05-29T21:48:19.884Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 181517
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = d59587dd-1a8c-4525-bb73-b60fb39c316f
response_code = 500
uri = /omicron-zones
|
| 1400 | 2024-05-29T21:48:19.884Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d59587dd-1a8c-4525-bb73-b60fb39c316f", "content-length": "124", "date": "Wed, 29 May 2024 21:48:19 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d59587dd-1a8c-4525-bb73-b60fb39c316f" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 9.821905465s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1401 | 2024-05-29T21:48:27.005Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 5.265240499s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:417
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 1402 | 2024-05-29T21:48:29.734Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1403 | 2024-05-29T21:48:29.734Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1404 | 2024-05-29T21:48:29.734Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1405 | 2024-05-29T21:48:29.755Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1406 | 2024-05-29T21:48:29.755Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1407 | 2024-05-29T21:48:29.773Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1408 | 2024-05-29T21:48:29.774Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1409 | 2024-05-29T21:48:29.792Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1410 | 2024-05-29T21:48:29.792Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1411 | 2024-05-29T21:48:29.810Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1412 | 2024-05-29T21:48:29.810Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1413 | 2024-05-29T21:48:29.829Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1414 | 2024-05-29T21:48:29.829Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1415 | 2024-05-29T21:48:29.848Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1416 | 2024-05-29T21:48:29.848Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1417 | 2024-05-29T21:48:29.866Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1418 | 2024-05-29T21:48:29.866Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1419 | 2024-05-29T21:48:29.883Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1420 | 2024-05-29T21:48:29.883Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1421 | 2024-05-29T21:48:29.883Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1422 | 2024-05-29T21:48:29.884Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1423 | 2024-05-29T21:48:29.914Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1424 | 2024-05-29T21:48:29.914Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1425 | 2024-05-29T21:48:29.914Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1426 | 2024-05-29T21:48:29.914Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1427 | 2024-05-29T21:48:29.914Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1428 | 2024-05-29T21:48:29.914Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized), ("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 179763
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = 7a42dab2-1296-4d54-91a4-b500899e8448
response_code = 500
uri = /omicron-zones
|
| 1429 | 2024-05-29T21:48:29.915Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "7a42dab2-1296-4d54-91a4-b500899e8448", "content-length": "124", "date": "Wed, 29 May 2024 21:48:29 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "7a42dab2-1296-4d54-91a4-b500899e8448" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 16.568318458s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1430 | 2024-05-29T21:48:40.778Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56128
|
| 1431 | 2024-05-29T21:48:40.778Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 31
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56128
req_id = 83a145b7-951c-4242-aa3a-46012b79524f
response_code = 200
uri = /baseboard
|
| 1432 | 2024-05-29T21:48:46.528Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1433 | 2024-05-29T21:48:46.529Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1434 | 2024-05-29T21:48:46.529Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1435 | 2024-05-29T21:48:46.551Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1436 | 2024-05-29T21:48:46.551Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1437 | 2024-05-29T21:48:46.570Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1438 | 2024-05-29T21:48:46.570Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1439 | 2024-05-29T21:48:46.588Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1440 | 2024-05-29T21:48:46.588Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1441 | 2024-05-29T21:48:46.608Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1442 | 2024-05-29T21:48:46.608Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1443 | 2024-05-29T21:48:46.626Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1444 | 2024-05-29T21:48:46.626Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1445 | 2024-05-29T21:48:46.645Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1446 | 2024-05-29T21:48:46.645Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1447 | 2024-05-29T21:48:46.662Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1448 | 2024-05-29T21:48:46.662Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1449 | 2024-05-29T21:48:46.679Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1450 | 2024-05-29T21:48:46.679Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1451 | 2024-05-29T21:48:46.680Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1452 | 2024-05-29T21:48:46.680Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1453 | 2024-05-29T21:48:46.710Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1454 | 2024-05-29T21:48:46.710Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1455 | 2024-05-29T21:48:46.710Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1456 | 2024-05-29T21:48:46.711Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1457 | 2024-05-29T21:48:46.711Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1458 | 2024-05-29T21:48:46.711Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014", TimeNotSynchronized), ("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54", TimeNotSynchronized), ("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b", TimeNotSynchronized), ("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a", TimeNotSynchronized), ("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 181553
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = 459fdba1-757b-4fe1-a413-4c57313b63e7
response_code = 500
uri = /omicron-zones
|
| 1459 | 2024-05-29T21:48:46.711Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "459fdba1-757b-4fe1-a413-4c57313b63e7", "content-length": "124", "date": "Wed, 29 May 2024 21:48:46 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "459fdba1-757b-4fe1-a413-4c57313b63e7" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 9.638413455s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1460 | 2024-05-29T21:48:47.364Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 5.220970111s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:417
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 1461 | 2024-05-29T21:48:56.377Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1462 | 2024-05-29T21:48:56.377Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1463 | 2024-05-29T21:48:56.377Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1464 | 2024-05-29T21:48:56.399Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1465 | 2024-05-29T21:48:56.399Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1466 | 2024-05-29T21:48:56.418Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1467 | 2024-05-29T21:48:56.418Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1468 | 2024-05-29T21:48:56.436Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1469 | 2024-05-29T21:48:56.436Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1470 | 2024-05-29T21:48:56.454Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1471 | 2024-05-29T21:48:56.454Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1472 | 2024-05-29T21:48:56.473Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1473 | 2024-05-29T21:48:56.473Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1474 | 2024-05-29T21:48:56.491Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1475 | 2024-05-29T21:48:56.491Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1476 | 2024-05-29T21:48:56.509Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1477 | 2024-05-29T21:48:56.509Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1478 | 2024-05-29T21:48:56.527Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1479 | 2024-05-29T21:48:56.527Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1480 | 2024-05-29T21:48:56.527Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1481 | 2024-05-29T21:48:56.528Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1482 | 2024-05-29T21:48:56.609Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1483 | 2024-05-29T21:48:56.609Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1484 | 2024-05-29T21:48:56.609Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1485 | 2024-05-29T21:48:56.609Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1486 | 2024-05-29T21:48:56.609Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1487 | 2024-05-29T21:48:56.699Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
file = illumos-utils/src/zone.rs:315
|
| 1488 | 2024-05-29T21:48:56.703Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
file = illumos-utils/src/zone.rs:315
|
| 1489 | 2024-05-29T21:48:56.707Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
file = illumos-utils/src/zone.rs:315
|
| 1490 | 2024-05-29T21:48:56.711Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
file = illumos-utils/src/zone.rs:315
|
| 1491 | 2024-05-29T21:48:56.716Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
file = illumos-utils/src/zone.rs:315
|
| 1492 | 2024-05-29T21:48:56.721Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
file = illumos-utils/src/zone.rs:354
|
| 1493 | 2024-05-29T21:48:56.725Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
file = illumos-utils/src/zone.rs:354
|
| 1494 | 2024-05-29T21:48:56.730Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
file = illumos-utils/src/zone.rs:354
|
| 1495 | 2024-05-29T21:48:56.734Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
file = illumos-utils/src/zone.rs:354
|
| 1496 | 2024-05-29T21:48:56.739Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
file = illumos-utils/src/zone.rs:354
|
| 1497 | 2024-05-29T21:48:59.558Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService6"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::3"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::3"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:1::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1498 | 2024-05-29T21:48:59.558Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1499 | 2024-05-29T21:48:59.564Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService5"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::4"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::4"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:1::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1500 | 2024-05-29T21:48:59.564Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1501 | 2024-05-29T21:48:59.582Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService7"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::6"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::6"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:1::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1502 | 2024-05-29T21:48:59.582Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1503 | 2024-05-29T21:48:59.600Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService8"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::5"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:1::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1504 | 2024-05-29T21:48:59.601Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1505 | 2024-05-29T21:48:59.622Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService9"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::7"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::7"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:1::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1506 | 2024-05-29T21:48:59.623Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1507 | 2024-05-29T21:48:59.714Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54") failed: Property not found. retry in 29.094859ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1508 | 2024-05-29T21:48:59.722Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe") failed: Property not found. retry in 33.917784ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1509 | 2024-05-29T21:48:59.729Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014") failed: Property not found. retry in 55.620559ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1510 | 2024-05-29T21:48:59.736Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 36.340373ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1511 | 2024-05-29T21:48:59.751Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54") failed: Property not found. retry in 60.601726ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1512 | 2024-05-29T21:48:59.758Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 47.763806ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1513 | 2024-05-29T21:48:59.765Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe") failed: Property not found. retry in 100.042064ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1514 | 2024-05-29T21:48:59.785Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 125.958493ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1515 | 2024-05-29T21:48:59.888Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014") failed: Property not found. retry in 133.305945ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1516 | 2024-05-29T21:49:00.074Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe") failed: Property not found. retry in 245.169946ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1517 | 2024-05-29T21:49:00.279Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54") failed: Property not found. retry in 172.100539ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1518 | 2024-05-29T21:49:00.511Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 100.259762ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1519 | 2024-05-29T21:49:00.617Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe") failed: Property not found. retry in 406.913864ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1520 | 2024-05-29T21:49:00.690Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54") failed: Property not found. retry in 363.250998ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1521 | 2024-05-29T21:49:00.737Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014") failed: Property not found. retry in 110.865106ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1522 | 2024-05-29T21:49:00.828Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 101.365375ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1523 | 2024-05-29T21:49:01.182Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 205.967874ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1524 | 2024-05-29T21:49:01.336Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe") failed: Property not found. retry in 733.919479ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1525 | 2024-05-29T21:49:01.887Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54") failed: Property not found. retry in 408.12299ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1526 | 2024-05-29T21:49:02.681Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014") failed: Property not found. retry in 367.17017ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1527 | 2024-05-29T21:49:02.765Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 408.081759ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1528 | 2024-05-29T21:49:02.774Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 251.289391ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1529 | 2024-05-29T21:49:02.782Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe") failed: Property not found. retry in 803.664773ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1530 | 2024-05-29T21:49:02.790Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54") failed: Property not found. retry in 514.074146ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1531 | 2024-05-29T21:49:03.037Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 1.009311829s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1532 | 2024-05-29T21:49:03.059Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014") failed: Property not found. retry in 437.199011ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1533 | 2024-05-29T21:49:03.186Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 1.04081541s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1534 | 2024-05-29T21:49:03.318Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54") failed: Property not found. retry in 1.409953295s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1535 | 2024-05-29T21:49:03.508Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014") failed: Property not found. retry in 856.015783ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1536 | 2024-05-29T21:49:03.598Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe") failed: Property not found. retry in 1.140985782s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1537 | 2024-05-29T21:49:04.056Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 1.351961621s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1538 | 2024-05-29T21:49:04.237Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 655.870187ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1539 | 2024-05-29T21:49:04.374Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014") failed: Property not found. retry in 844.199758ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1540 | 2024-05-29T21:49:04.738Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54") failed: Property not found. retry in 1.121547017s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1541 | 2024-05-29T21:49:04.747Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe") failed: Property not found. retry in 1.311643632s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1542 | 2024-05-29T21:49:04.901Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 1.267263076s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1543 | 2024-05-29T21:49:05.228Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014") failed: Property not found. retry in 1.186346455s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1544 | 2024-05-29T21:49:05.418Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 547.065454ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1545 | 2024-05-29T21:49:05.870Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54") failed: Property not found. retry in 1.249065249s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1546 | 2024-05-29T21:49:05.975Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 1.022376493s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1547 | 2024-05-29T21:49:06.070Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe") failed: Property not found. retry in 1.217803767s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1548 | 2024-05-29T21:49:06.179Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 995.336328ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1549 | 2024-05-29T21:49:06.426Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014") failed: Property not found. retry in 820.875042ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1550 | 2024-05-29T21:49:07.006Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 1.18259569s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1551 | 2024-05-29T21:49:07.128Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54") failed: Property not found. retry in 962.820449ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1552 | 2024-05-29T21:49:07.183Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 592.614948ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1553 | 2024-05-29T21:49:07.256Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014") failed: Property not found. retry in 742.688196ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1554 | 2024-05-29T21:49:07.297Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe") failed: Property not found. retry in 1.100238358s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1555 | 2024-05-29T21:49:07.663Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 9.926976148s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:417
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 1556 | 2024-05-29T21:49:07.786Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 1.170297281s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1557 | 2024-05-29T21:49:08.008Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014") failed: Property not found. retry in 1.412545362s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1558 | 2024-05-29T21:49:08.101Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54") failed: Property not found. retry in 789.986132ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1559 | 2024-05-29T21:49:08.198Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 552.529583ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1560 | 2024-05-29T21:49:08.406Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe") failed: Property not found. retry in 813.388612ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1561 | 2024-05-29T21:49:08.761Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 848.490632ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1562 | 2024-05-29T21:49:08.899Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54") failed: Property not found. retry in 1.447961161s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1563 | 2024-05-29T21:49:08.966Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 514.27182ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1564 | 2024-05-29T21:49:09.230Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe") failed: Property not found. retry in 1.441644369s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1565 | 2024-05-29T21:49:09.430Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014") failed: Property not found. retry in 1.440714611s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1566 | 2024-05-29T21:49:09.490Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 839.43301ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1567 | 2024-05-29T21:49:09.619Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 1.009835659s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1568 | 2024-05-29T21:49:10.341Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 1.13138345s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1569 | 2024-05-29T21:49:10.639Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 903.729518ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1570 | 2024-05-29T21:49:10.885Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:60796
|
| 1571 | 2024-05-29T21:49:10.885Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 112
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:60796
req_id = cfe2ff1b-2414-4647-8283-42f6e0f10a68
response_code = 200
uri = /baseboard
|
| 1572 | 2024-05-29T21:49:11.483Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 1.042496911s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1573 | 2024-05-29T21:49:11.551Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 890.539951ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1574 | 2024-05-29T21:49:12.453Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 1.382973959s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1575 | 2024-05-29T21:49:12.535Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 1.215451534s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1576 | 2024-05-29T21:49:14.226Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 1.045106203s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1577 | 2024-05-29T21:49:14.240Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 974.693586ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1578 | 2024-05-29T21:49:15.227Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 1.281713596s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1579 | 2024-05-29T21:49:15.282Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 1.173277393s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1580 | 2024-05-29T21:49:16.465Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 717.341259ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1581 | 2024-05-29T21:49:16.519Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 1.405818359s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1582 | 2024-05-29T21:49:17.192Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 1.076140837s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1583 | 2024-05-29T21:49:17.934Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 1.129258951s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1584 | 2024-05-29T21:49:18.278Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 779.004634ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1585 | 2024-05-29T21:49:19.066Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b") failed: Property not found. retry in 647.220671ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1586 | 2024-05-29T21:49:19.074Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a") failed: Property not found. retry in 930.530209ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1587 | 2024-05-29T21:49:20.025Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe
|
| 1588 | 2024-05-29T21:49:20.025Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54
|
| 1589 | 2024-05-29T21:49:20.025Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014
|
| 1590 | 2024-05-29T21:49:20.025Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b
|
| 1591 | 2024-05-29T21:49:20.025Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a
|
| 1592 | 2024-05-29T21:49:20.025Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 1593 | 2024-05-29T21:49:20.025Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 1594 | 2024-05-29T21:49:20.026Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 23631122
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:43427
req_id = a1f5e8b5-49ad-4e2b-b0ae-86e3a0c3e9a8
response_code = 204
uri = /omicron-zones
|
| 1595 | 2024-05-29T21:49:20.057Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:63868
|
| 1596 | 2024-05-29T21:49:20.087Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 29204
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = e5c66b3e-a48c-4039-bc01-daa2646bc97e
response_code = 500
uri = /cockroachdb
|
| 1597 | 2024-05-29T21:49:20.087Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e5c66b3e-a48c-4039-bc01-daa2646bc97e", "content-length": "124", "date": "Wed, 29 May 2024 21:49:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e5c66b3e-a48c-4039-bc01-daa2646bc97e" }
file = sled-agent/src/rack_setup/service.rs:958
retry_after = 67.280661ms
|
| 1598 | 2024-05-29T21:49:20.183Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 27075
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = 41a7cad8-0883-418e-8063-53ef085ac5ae
response_code = 500
uri = /cockroachdb
|
| 1599 | 2024-05-29T21:49:20.183Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "41a7cad8-0883-418e-8063-53ef085ac5ae", "content-length": "124", "date": "Wed, 29 May 2024 21:49:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "41a7cad8-0883-418e-8063-53ef085ac5ae" }
file = sled-agent/src/rack_setup/service.rs:958
retry_after = 139.876271ms
|
| 1600 | 2024-05-29T21:49:20.351Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 26044
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = f92329e5-dd0a-4b99-96cf-ecd2a11f7930
response_code = 500
uri = /cockroachdb
|
| 1601 | 2024-05-29T21:49:20.351Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f92329e5-dd0a-4b99-96cf-ecd2a11f7930", "content-length": "124", "date": "Wed, 29 May 2024 21:49:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f92329e5-dd0a-4b99-96cf-ecd2a11f7930" }
file = sled-agent/src/rack_setup/service.rs:958
retry_after = 121.005904ms
|
| 1602 | 2024-05-29T21:49:20.499Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 25758
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = af9fa4a9-4eea-4033-8b4d-32d2898946c1
response_code = 500
uri = /cockroachdb
|
| 1603 | 2024-05-29T21:49:20.499Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "af9fa4a9-4eea-4033-8b4d-32d2898946c1", "content-length": "124", "date": "Wed, 29 May 2024 21:49:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "af9fa4a9-4eea-4033-8b4d-32d2898946c1" }
file = sled-agent/src/rack_setup/service.rs:958
retry_after = 216.108364ms
|
| 1604 | 2024-05-29T21:49:20.745Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 28300
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = f9710943-e826-42ec-920a-1eea347aeeb5
response_code = 500
uri = /cockroachdb
|
| 1605 | 2024-05-29T21:49:20.745Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f9710943-e826-42ec-920a-1eea347aeeb5", "content-length": "124", "date": "Wed, 29 May 2024 21:49:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f9710943-e826-42ec-920a-1eea347aeeb5" }
file = sled-agent/src/rack_setup/service.rs:958
retry_after = 137.328199ms
|
| 1606 | 2024-05-29T21:49:20.964Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 75404
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = e8a69e5c-9b4d-486d-b6e4-4d4ad45ce1ff
response_code = 500
uri = /cockroachdb
|
| 1607 | 2024-05-29T21:49:20.964Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e8a69e5c-9b4d-486d-b6e4-4d4ad45ce1ff", "content-length": "124", "date": "Wed, 29 May 2024 21:49:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e8a69e5c-9b4d-486d-b6e4-4d4ad45ce1ff" }
file = sled-agent/src/rack_setup/service.rs:958
retry_after = 344.608997ms
|
| 1608 | 2024-05-29T21:49:21.340Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 30456
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = 2c722084-c176-45c0-b337-acc822bb1e75
response_code = 500
uri = /cockroachdb
|
| 1609 | 2024-05-29T21:49:21.340Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "2c722084-c176-45c0-b337-acc822bb1e75", "content-length": "124", "date": "Wed, 29 May 2024 21:49:21 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "2c722084-c176-45c0-b337-acc822bb1e75" }
file = sled-agent/src/rack_setup/service.rs:958
retry_after = 212.686903ms
|
| 1610 | 2024-05-29T21:49:21.584Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 29735
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = fef3a5f3-6f0c-4ed6-b000-06a15d25d282
response_code = 500
uri = /cockroachdb
|
| 1611 | 2024-05-29T21:49:21.585Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "fef3a5f3-6f0c-4ed6-b000-06a15d25d282", "content-length": "124", "date": "Wed, 29 May 2024 21:49:21 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "fef3a5f3-6f0c-4ed6-b000-06a15d25d282" }
file = sled-agent/src/rack_setup/service.rs:958
retry_after = 350.448905ms
|
| 1612 | 2024-05-29T21:49:21.965Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 28126
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = 4067392c-12c1-4f72-8cbf-87c5f2452722
response_code = 500
uri = /cockroachdb
|
| 1613 | 2024-05-29T21:49:21.965Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "4067392c-12c1-4f72-8cbf-87c5f2452722", "content-length": "124", "date": "Wed, 29 May 2024 21:49:21 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "4067392c-12c1-4f72-8cbf-87c5f2452722" }
file = sled-agent/src/rack_setup/service.rs:958
retry_after = 266.571005ms
|
| 1614 | 2024-05-29T21:49:22.259Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 26910
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = e71e42fd-a25c-4933-a584-a74dfb6d0868
response_code = 500
uri = /cockroachdb
|
| 1615 | 2024-05-29T21:49:22.259Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e71e42fd-a25c-4933-a584-a74dfb6d0868", "content-length": "124", "date": "Wed, 29 May 2024 21:49:21 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e71e42fd-a25c-4933-a584-a74dfb6d0868" }
file = sled-agent/src/rack_setup/service.rs:958
retry_after = 404.480192ms
|
| 1616 | 2024-05-29T21:49:22.692Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 27509
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = b5cb2f89-3b71-4617-8c2b-5df3af8d3c81
response_code = 500
uri = /cockroachdb
|
| 1617 | 2024-05-29T21:49:22.693Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b5cb2f89-3b71-4617-8c2b-5df3af8d3c81", "content-length": "124", "date": "Wed, 29 May 2024 21:49:22 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b5cb2f89-3b71-4617-8c2b-5df3af8d3c81" }
file = sled-agent/src/rack_setup/service.rs:958
retry_after = 622.973441ms
|
| 1618 | 2024-05-29T21:49:23.344Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 27473
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = a66ee7ff-7cd5-4701-b148-bea2144f61e4
response_code = 500
uri = /cockroachdb
|
| 1619 | 2024-05-29T21:49:23.344Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a66ee7ff-7cd5-4701-b148-bea2144f61e4", "content-length": "124", "date": "Wed, 29 May 2024 21:49:22 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a66ee7ff-7cd5-4701-b148-bea2144f61e4" }
file = sled-agent/src/rack_setup/service.rs:958
retry_after = 618.718136ms
|
| 1620 | 2024-05-29T21:49:23.992Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 28451
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = 389a2393-aad1-4c99-86c9-fbc9d9d724bb
response_code = 500
uri = /cockroachdb
|
| 1621 | 2024-05-29T21:49:23.992Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "389a2393-aad1-4c99-86c9-fbc9d9d724bb", "content-length": "124", "date": "Wed, 29 May 2024 21:49:23 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "389a2393-aad1-4c99-86c9-fbc9d9d724bb" }
file = sled-agent/src/rack_setup/service.rs:958
retry_after = 734.650751ms
|
| 1622 | 2024-05-29T21:49:24.753Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 25583
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = 6795d49d-4c1b-47fc-828d-d36cb9529ccc
response_code = 500
uri = /cockroachdb
|
| 1623 | 2024-05-29T21:49:24.753Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6795d49d-4c1b-47fc-828d-d36cb9529ccc", "content-length": "124", "date": "Wed, 29 May 2024 21:49:23 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6795d49d-4c1b-47fc-828d-d36cb9529ccc" }
file = sled-agent/src/rack_setup/service.rs:958
retry_after = 1.25944816s
|
| 1624 | 2024-05-29T21:49:26.043Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221
file = sled-agent/src/services.rs:3437
|
| 1625 | 2024-05-29T21:49:34.275Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3455
|
| 1626 | 2024-05-29T21:49:41.776Z | INFO | SledAgent (ServiceManager): Formatting CRDB - Completed
file = sled-agent/src/services.rs:3478
|
| 1627 | 2024-05-29T21:49:41.776Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 15762170
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:63868
req_id = 1dc7c552-64b4-4f38-b7e8-65700c417178
response_code = 204
uri = /cockroachdb
|
| 1628 | 2024-05-29T21:49:41.776Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56476
|
| 1629 | 2024-05-29T21:49:41.777Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 197
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56476
req_id = eff9c2ce-24ec-4a71-b5f5-7fa05d13398e
response_code = 200
uri = /baseboard
|
| 1630 | 2024-05-29T21:49:41.781Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 26.741615336s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:417
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 1631 | 2024-05-29T21:49:41.811Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name("ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }, OmicronZoneConfig { id: ce802882-5f60-4fa0-8546-381dc464eec5, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name("external-dns-ce802882-5f60-4fa0-8546-381dc464eec5"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name("external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name("nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 73fd9e9c-3929-404c-966e-fc226ac774de, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name("nexus-73fd9e9c-3929-404c-966e-fc226ac774de"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name("nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b0594198-29c3-400f-bbd5-0154b6bd4027, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 01234475-c550-4426-ba63-e228db75b110, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: 03b95682-bf8e-4d86-bca9-1c8acc0d4e8c, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 91d995f5-4121-4031-8f6b-37b0a19cd7a2, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 4287d8cf-0565-4f0e-abd8-12de702d3848, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9982d53a-69ab-47ac-b36a-154a6b0da765, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: d462ebe6-9d90-445d-bf8a-7c34338191d9, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7e9ce69-0abf-4995-ace8-8f9a44bb9c50, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: a9a35d48-4126-4b06-b5c7-ee36d97697af, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: 1651e1ef-46f8-46cf-a194-eed8a22e18b8, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }, OmicronZoneConfig { id: 5de4c38e-911d-46b7-beee-a39576f2ce66, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b73c1d9-c688-4821-bfd4-1080d5b2eea5, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool), kind: External } } } }, OmicronZoneConfig { id: 210fd0f2-c8d5-4a6d-a520-1ba68096679b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool), kind: External } } } }, OmicronZoneConfig { id: d3e79ddb-29ef-48e6-9b00-cd52126fcc80, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b416693-d467-424a-86a1-703b05ad3101 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1632 | 2024-05-29T21:49:41.811Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:47072
|
| 1633 | 2024-05-29T21:49:41.811Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1634 | 2024-05-29T21:49:41.811Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1635 | 2024-05-29T21:49:41.836Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1636 | 2024-05-29T21:49:41.836Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1637 | 2024-05-29T21:49:41.856Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1638 | 2024-05-29T21:49:41.856Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1639 | 2024-05-29T21:49:41.877Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1640 | 2024-05-29T21:49:41.877Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1641 | 2024-05-29T21:49:41.897Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1642 | 2024-05-29T21:49:41.897Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1643 | 2024-05-29T21:49:41.916Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1644 | 2024-05-29T21:49:41.916Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1645 | 2024-05-29T21:49:41.936Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1646 | 2024-05-29T21:49:41.936Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1647 | 2024-05-29T21:49:41.956Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1648 | 2024-05-29T21:49:41.956Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
| 1649 | 2024-05-29T21:49:41.974Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ce802882-5f60-4fa0-8546-381dc464eec5, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1650 | 2024-05-29T21:49:41.974Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ce802882-5f60-4fa0-8546-381dc464eec5, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1651 | 2024-05-29T21:49:42.080Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1652 | 2024-05-29T21:49:42.080Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1653 | 2024-05-29T21:49:42.191Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 01234475-c550-4426-ba63-e228db75b110, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1654 | 2024-05-29T21:49:42.191Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 01234475-c550-4426-ba63-e228db75b110, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1655 | 2024-05-29T21:49:42.367Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9982d53a-69ab-47ac-b36a-154a6b0da765, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1656 | 2024-05-29T21:49:42.367Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9982d53a-69ab-47ac-b36a-154a6b0da765, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1657 | 2024-05-29T21:49:42.483Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d462ebe6-9d90-445d-bf8a-7c34338191d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1658 | 2024-05-29T21:49:42.483Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d462ebe6-9d90-445d-bf8a-7c34338191d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1659 | 2024-05-29T21:49:42.724Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d7e9ce69-0abf-4995-ace8-8f9a44bb9c50, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1660 | 2024-05-29T21:49:42.724Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d7e9ce69-0abf-4995-ace8-8f9a44bb9c50, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1661 | 2024-05-29T21:49:42.822Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a9a35d48-4126-4b06-b5c7-ee36d97697af, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1662 | 2024-05-29T21:49:42.822Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: a9a35d48-4126-4b06-b5c7-ee36d97697af, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1663 | 2024-05-29T21:49:42.934Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1651e1ef-46f8-46cf-a194-eed8a22e18b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1664 | 2024-05-29T21:49:42.934Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 1651e1ef-46f8-46cf-a194-eed8a22e18b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1665 | 2024-05-29T21:49:43.029Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5de4c38e-911d-46b7-beee-a39576f2ce66, dataset_name: DatasetName { pool_name: ZpoolName { id: 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1666 | 2024-05-29T21:49:43.029Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 5de4c38e-911d-46b7-beee-a39576f2ce66, dataset_name: DatasetName { pool_name: ZpoolName { id: 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1667 | 2024-05-29T21:49:43.131Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b73c1d9-c688-4821-bfd4-1080d5b2eea5, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1668 | 2024-05-29T21:49:43.131Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2b73c1d9-c688-4821-bfd4-1080d5b2eea5, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1669 | 2024-05-29T21:49:43.229Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 210fd0f2-c8d5-4a6d-a520-1ba68096679b, dataset_name: DatasetName { pool_name: ZpoolName { id: 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1670 | 2024-05-29T21:49:43.230Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 210fd0f2-c8d5-4a6d-a520-1ba68096679b, dataset_name: DatasetName { pool_name: ZpoolName { id: 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1671 | 2024-05-29T21:49:43.322Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d3e79ddb-29ef-48e6-9b00-cd52126fcc80, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b416693-d467-424a-86a1-703b05ad3101 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
| 1672 | 2024-05-29T21:49:43.322Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d3e79ddb-29ef-48e6-9b00-cd52126fcc80, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b416693-d467-424a-86a1-703b05ad3101 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1673 | 2024-05-29T21:49:43.403Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1674 | 2024-05-29T21:49:43.403Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1675 | 2024-05-29T21:49:43.403Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1676 | 2024-05-29T21:49:43.404Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1677 | 2024-05-29T21:49:43.523Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:259
|
| 1678 | 2024-05-29T21:49:43.559Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:259
|
| 1679 | 2024-05-29T21:49:43.559Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1680 | 2024-05-29T21:49:43.559Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1681 | 2024-05-29T21:49:43.560Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:259
|
| 1682 | 2024-05-29T21:49:43.562Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:259
|
| 1683 | 2024-05-29T21:49:43.602Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1684 | 2024-05-29T21:49:43.603Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1685 | 2024-05-29T21:49:43.603Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1686 | 2024-05-29T21:49:43.603Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1687 | 2024-05-29T21:49:43.603Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1688 | 2024-05-29T21:49:43.788Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:259
|
| 1689 | 2024-05-29T21:49:43.788Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1690 | 2024-05-29T21:49:43.788Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1691 | 2024-05-29T21:49:43.788Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1692 | 2024-05-29T21:49:43.788Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1693 | 2024-05-29T21:49:43.812Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1694 | 2024-05-29T21:49:43.812Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1695 | 2024-05-29T21:49:43.812Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1696 | 2024-05-29T21:49:43.813Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
file = illumos-utils/src/zone.rs:315
|
| 1697 | 2024-05-29T21:49:43.818Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
file = illumos-utils/src/zone.rs:315
|
| 1698 | 2024-05-29T21:49:43.824Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
file = illumos-utils/src/zone.rs:315
|
| 1699 | 2024-05-29T21:49:43.829Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
file = illumos-utils/src/zone.rs:315
|
| 1700 | 2024-05-29T21:49:43.834Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
file = illumos-utils/src/zone.rs:315
|
| 1701 | 2024-05-29T21:49:43.839Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
file = illumos-utils/src/zone.rs:315
|
| 1702 | 2024-05-29T21:49:43.843Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
file = illumos-utils/src/zone.rs:315
|
| 1703 | 2024-05-29T21:49:43.976Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
file = illumos-utils/src/zone.rs:354
|
| 1704 | 2024-05-29T21:49:43.980Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
file = illumos-utils/src/zone.rs:354
|
| 1705 | 2024-05-29T21:49:43.984Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
file = illumos-utils/src/zone.rs:354
|
| 1706 | 2024-05-29T21:49:43.988Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
file = illumos-utils/src/zone.rs:354
|
| 1707 | 2024-05-29T21:49:43.993Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
file = illumos-utils/src/zone.rs:354
|
| 1708 | 2024-05-29T21:49:43.997Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
file = illumos-utils/src/zone.rs:354
|
| 1709 | 2024-05-29T21:49:44.002Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
file = illumos-utils/src/zone.rs:354
|
| 1710 | 2024-05-29T21:49:44.007Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
file = illumos-utils/src/zone.rs:315
|
| 1711 | 2024-05-29T21:49:44.013Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
file = illumos-utils/src/zone.rs:315
|
| 1712 | 2024-05-29T21:49:44.018Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
file = illumos-utils/src/zone.rs:315
|
| 1713 | 2024-05-29T21:49:44.023Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
file = illumos-utils/src/zone.rs:315
|
| 1714 | 2024-05-29T21:49:44.028Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
file = illumos-utils/src/zone.rs:315
|
| 1715 | 2024-05-29T21:49:44.034Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
file = illumos-utils/src/zone.rs:315
|
| 1716 | 2024-05-29T21:49:44.039Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
file = illumos-utils/src/zone.rs:315
|
| 1717 | 2024-05-29T21:49:44.045Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
file = illumos-utils/src/zone.rs:354
|
| 1718 | 2024-05-29T21:49:44.051Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
file = illumos-utils/src/zone.rs:354
|
| 1719 | 2024-05-29T21:49:44.057Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
file = illumos-utils/src/zone.rs:354
|
| 1720 | 2024-05-29T21:49:44.062Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
file = illumos-utils/src/zone.rs:354
|
| 1721 | 2024-05-29T21:49:44.067Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
file = illumos-utils/src/zone.rs:354
|
| 1722 | 2024-05-29T21:49:44.072Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
file = illumos-utils/src/zone.rs:354
|
| 1723 | 2024-05-29T21:49:44.077Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
file = illumos-utils/src/zone.rs:354
|
| 1724 | 2024-05-29T21:49:44.134Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:276
|
| 1725 | 2024-05-29T21:49:44.174Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:276
|
| 1726 | 2024-05-29T21:49:44.211Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:276
|
| 1727 | 2024-05-29T21:49:44.248Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:276
|
| 1728 | 2024-05-29T21:49:44.358Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:276
|
| 1729 | 2024-05-29T21:49:44.360Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:301
|
| 1730 | 2024-05-29T21:49:44.360Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:193
|
| 1731 | 2024-05-29T21:49:44.444Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte1", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 201, 64]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte1" } }
|
| 1732 | 2024-05-29T21:49:44.444Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = nexus
|
| 1733 | 2024-05-29T21:49:44.448Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:301
|
| 1734 | 2024-05-29T21:49:44.448Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:193
|
| 1735 | 2024-05-29T21:49:44.565Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte2", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 177, 61]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: "vopte2" } }
|
| 1736 | 2024-05-29T21:49:44.565Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = external_dns
|
| 1737 | 2024-05-29T21:49:44.569Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:301
|
| 1738 | 2024-05-29T21:49:44.569Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:193
|
| 1739 | 2024-05-29T21:49:44.668Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte3", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 190, 130]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte3" } }
|
| 1740 | 2024-05-29T21:49:44.668Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = nexus
|
| 1741 | 2024-05-29T21:49:44.671Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:301
|
| 1742 | 2024-05-29T21:49:44.671Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:193
|
| 1743 | 2024-05-29T21:49:44.776Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte4", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 157, 246]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte4" } }
|
| 1744 | 2024-05-29T21:49:44.776Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = nexus
|
| 1745 | 2024-05-29T21:49:44.780Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:301
|
| 1746 | 2024-05-29T21:49:44.780Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:193
|
| 1747 | 2024-05-29T21:49:44.875Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte5", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 234, 2]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: "vopte5" } }
|
| 1748 | 2024-05-29T21:49:44.875Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = external_dns
|
| 1749 | 2024-05-29T21:49:44.876Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 1750 | 2024-05-29T21:49:44.876Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 1751 | 2024-05-29T21:49:44.876Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 1752 | 2024-05-29T21:49:44.876Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 1753 | 2024-05-29T21:49:44.877Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 1754 | 2024-05-29T21:49:44.877Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 1755 | 2024-05-29T21:49:44.877Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 1756 | 2024-05-29T21:49:44.877Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 1757 | 2024-05-29T21:49:44.878Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 1758 | 2024-05-29T21:49:44.881Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 1759 | 2024-05-29T21:49:44.884Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1760 | 2024-05-29T21:49:44.884Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 1761 | 2024-05-29T21:49:44.884Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1762 | 2024-05-29T21:49:44.889Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 1763 | 2024-05-29T21:49:44.892Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1764 | 2024-05-29T21:49:44.892Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1765 | 2024-05-29T21:49:45.058Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
| 1766 | 2024-05-29T21:49:45.058Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
| 1767 | 2024-05-29T21:49:45.059Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
file = illumos-utils/src/zone.rs:315
|
| 1768 | 2024-05-29T21:49:45.069Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
file = illumos-utils/src/zone.rs:315
|
| 1769 | 2024-05-29T21:49:45.079Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
file = illumos-utils/src/zone.rs:315
|
| 1770 | 2024-05-29T21:49:45.090Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
file = illumos-utils/src/zone.rs:315
|
| 1771 | 2024-05-29T21:49:45.101Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
| 1772 | 2024-05-29T21:49:45.104Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 1773 | 2024-05-29T21:49:45.104Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
file = illumos-utils/src/zone.rs:354
|
| 1774 | 2024-05-29T21:49:45.114Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
file = illumos-utils/src/zone.rs:354
|
| 1775 | 2024-05-29T21:49:45.122Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
file = illumos-utils/src/zone.rs:354
|
| 1776 | 2024-05-29T21:49:45.129Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
file = illumos-utils/src/zone.rs:354
|
| 1777 | 2024-05-29T21:49:45.214Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
file = illumos-utils/src/zone.rs:315
|
| 1778 | 2024-05-29T21:49:45.320Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
file = illumos-utils/src/zone.rs:354
|
| 1779 | 2024-05-29T21:49:47.083Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService11"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::17"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_4532d18b-4909-4537-9ce6-98b74fb663f5/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::17"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="af56d76f-2b01-458a-9eda-90425c7c7257"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1780 | 2024-05-29T21:49:47.083Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1781 | 2024-05-29T21:49:47.941Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService13"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::11"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::11"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1782 | 2024-05-29T21:49:47.942Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1783 | 2024-05-29T21:49:47.950Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService14"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::13"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_8f301520-a970-47c3-90c3-bd3006e5615c/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::13"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="8f13b51f-00e4-463a-9faf-243d14aac488"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1784 | 2024-05-29T21:49:47.950Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1785 | 2024-05-29T21:49:47.985Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService19"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::16"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_020b43a6-e481-4304-8596-25774e86f14b/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::16"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="f25aa5ca-03ce-4184-95ad-023c5bf18d9e"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1786 | 2024-05-29T21:49:47.985Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1787 | 2024-05-29T21:49:47.996Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService22"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::18"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_8d81843f-0d2d-4c69-8d17-45004d7cf484/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::18"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="d5779ea1-0d2b-4ba7-b720-5fca991a8372"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1788 | 2024-05-29T21:49:47.996Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1789 | 2024-05-29T21:49:48.022Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService16"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::12"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::12"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="a7870ac2-efb6-4f27-924c-8bc2266c1744"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1790 | 2024-05-29T21:49:48.023Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1791 | 2024-05-29T21:49:48.094Z | INFO | SledAgent (ServiceManager): Profile for oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService12"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::d"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/oximeter">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="id" value="b0594198-29c3-400f-bbd5-0154b6bd4027"/>
<propval type="astring" name="address" value="[fd00:1122:3344:101::d]:12223"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1792 | 2024-05-29T21:49:48.095Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1793 | 2024-05-29T21:49:48.132Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 44.315636ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1794 | 2024-05-29T21:49:48.137Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService15"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::14"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::14"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="8f5c15d4-49bf-4267-8f22-810ec65d76ee"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1795 | 2024-05-29T21:49:48.138Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 1796 | 2024-05-29T21:49:48.150Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService21"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::15"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_b5814935-58c2-4e3c-9137-4cbaa1ee9fab/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::15"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="7062bce1-7553-4351-a6c6-2bd4afd39de8"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1797 | 2024-05-29T21:49:48.150Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1798 | 2024-05-29T21:49:49.115Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 93.984579ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1799 | 2024-05-29T21:49:49.177Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService20"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::10"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::10"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1800 | 2024-05-29T21:49:49.183Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1801 | 2024-05-29T21:49:49.205Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 26.744865ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1802 | 2024-05-29T21:49:49.209Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService23"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::f"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::f"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1803 | 2024-05-29T21:49:49.209Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1804 | 2024-05-29T21:49:49.235Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 282.703936ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1805 | 2024-05-29T21:49:49.250Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 100.08836ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1806 | 2024-05-29T21:49:49.265Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 34.694986ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1807 | 2024-05-29T21:49:49.280Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 57.064798ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1808 | 2024-05-29T21:49:49.295Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 61.682419ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1809 | 2024-05-29T21:49:49.316Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 66.985499ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1810 | 2024-05-29T21:49:49.383Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 60.946399ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1811 | 2024-05-29T21:49:49.403Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 112.440188ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1812 | 2024-05-29T21:49:49.425Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 57.226477ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1813 | 2024-05-29T21:49:49.444Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 51.524639ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 1814 | 2024-05-29T21:49:49.461Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 49.779427ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1815 | 2024-05-29T21:49:49.461Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService17"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::19"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_59fcaf06-997c-43d9-9256-26a5eda3000f/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::19"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="5f152cde-fd85-48b0-8525-ef2f54cf2302"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1816 | 2024-05-29T21:49:49.461Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService18"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::1a"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_6b416693-d467-424a-86a1-703b05ad3101/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::1a"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="5d73d8c5-5993-48f8-9e08-8f3869dbd964"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1817 | 2024-05-29T21:49:49.476Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 136.727289ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1818 | 2024-05-29T21:49:49.491Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 68.015523ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1819 | 2024-05-29T21:49:50.545Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 40.90717ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1820 | 2024-05-29T21:49:50.622Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 147.799915ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1821 | 2024-05-29T21:49:50.651Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 202.450431ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1822 | 2024-05-29T21:49:50.684Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 132.750803ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1823 | 2024-05-29T21:49:50.738Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 328.425693ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1824 | 2024-05-29T21:49:50.808Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 211.090628ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1825 | 2024-05-29T21:49:50.843Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 51.952127ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 1826 | 2024-05-29T21:49:50.891Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 81.662528ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1827 | 2024-05-29T21:49:50.891Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1828 | 2024-05-29T21:49:50.904Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1829 | 2024-05-29T21:49:50.942Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 332.720604ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1830 | 2024-05-29T21:49:51.093Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 53.485232ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1831 | 2024-05-29T21:49:51.210Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 136.221838ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1832 | 2024-05-29T21:49:51.350Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 279.133607ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1833 | 2024-05-29T21:49:51.366Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 28.671829ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1834 | 2024-05-29T21:49:51.378Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 548.721152ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1835 | 2024-05-29T21:49:51.389Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 191.321264ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1836 | 2024-05-29T21:49:51.401Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 640.970966ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1837 | 2024-05-29T21:49:51.414Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 292.59192ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1838 | 2024-05-29T21:49:51.428Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 102.146905ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 1839 | 2024-05-29T21:49:51.440Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 298.410373ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1840 | 2024-05-29T21:49:51.451Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 520.133672ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1841 | 2024-05-29T21:49:51.463Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 185.123474ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1842 | 2024-05-29T21:49:51.474Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 283.582701ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1843 | 2024-05-29T21:49:51.484Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 145.914352ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1844 | 2024-05-29T21:49:51.493Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 59.891428ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1845 | 2024-05-29T21:49:51.524Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 33.982731ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1846 | 2024-05-29T21:49:51.541Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 406.82564ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 1847 | 2024-05-29T21:49:51.561Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 114.25379ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1848 | 2024-05-29T21:49:51.569Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 105.796042ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1849 | 2024-05-29T21:49:51.592Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 598.650504ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1850 | 2024-05-29T21:49:51.643Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 304.224174ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1851 | 2024-05-29T21:49:51.653Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 118.72515ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1852 | 2024-05-29T21:49:51.661Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 522.106964ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1853 | 2024-05-29T21:49:51.686Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 171.6522ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1854 | 2024-05-29T21:49:51.699Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 267.898091ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1855 | 2024-05-29T21:49:51.719Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 881.152628ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1856 | 2024-05-29T21:49:51.750Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 384.79976ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1857 | 2024-05-29T21:49:51.768Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 484.992526ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1858 | 2024-05-29T21:49:51.784Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 588.446499ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1859 | 2024-05-29T21:49:51.876Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 211.427669ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1860 | 2024-05-29T21:49:51.942Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 688.807679ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1861 | 2024-05-29T21:49:51.962Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 855.657097ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 1862 | 2024-05-29T21:49:51.976Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 569.074168ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1863 | 2024-05-29T21:49:51.988Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 424.69712ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1864 | 2024-05-29T21:49:52.002Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 1.486254385s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1865 | 2024-05-29T21:49:52.057Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 1.306974747s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1866 | 2024-05-29T21:49:52.101Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 772.654009ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1867 | 2024-05-29T21:49:52.151Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 1.021248748s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1868 | 2024-05-29T21:49:52.201Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 862.681335ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1869 | 2024-05-29T21:49:52.218Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 533.013546ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1870 | 2024-05-29T21:49:52.271Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 1.190909976s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1871 | 2024-05-29T21:49:52.388Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 450.289133ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1872 | 2024-05-29T21:49:52.435Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 449.10904ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1873 | 2024-05-29T21:49:52.561Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 1.203616531s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1874 | 2024-05-29T21:49:52.616Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 1.207956812s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1875 | 2024-05-29T21:49:52.647Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 568.333483ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1876 | 2024-05-29T21:49:52.766Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 1.247066435s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1877 | 2024-05-29T21:49:52.833Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 1.488477281s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 1878 | 2024-05-29T21:49:52.863Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 646.61591ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1879 | 2024-05-29T21:49:52.887Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 816.447648ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1880 | 2024-05-29T21:49:52.898Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 683.539956ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1881 | 2024-05-29T21:49:53.077Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 949.374259ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1882 | 2024-05-29T21:49:53.188Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 907.76871ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1883 | 2024-05-29T21:49:53.234Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 877.010332ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1884 | 2024-05-29T21:49:53.383Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 553.265064ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1885 | 2024-05-29T21:49:53.489Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 1.495902429s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1886 | 2024-05-29T21:49:56.715Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 588.693987ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1887 | 2024-05-29T21:49:56.741Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 955.654393ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1888 | 2024-05-29T21:49:56.762Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 1.136533636s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1889 | 2024-05-29T21:49:56.776Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 849.08357ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1890 | 2024-05-29T21:49:56.791Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 1.371887023s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1891 | 2024-05-29T21:49:56.802Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 1.082781557s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 1892 | 2024-05-29T21:49:56.813Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 1.033965031s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1893 | 2024-05-29T21:49:56.825Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 861.9371ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1894 | 2024-05-29T21:49:56.839Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 535.92412ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1895 | 2024-05-29T21:49:56.850Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 598.23658ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1896 | 2024-05-29T21:49:56.862Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 1.212587778s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1897 | 2024-05-29T21:49:56.874Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 1.127237468s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1898 | 2024-05-29T21:49:56.887Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 1.048557141s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1899 | 2024-05-29T21:49:57.319Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 1.373428231s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1900 | 2024-05-29T21:49:57.390Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 1.065219848s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1901 | 2024-05-29T21:49:57.462Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 1.44209924s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1902 | 2024-05-29T21:49:57.639Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 1.474996452s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1903 | 2024-05-29T21:49:57.701Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 1.46683993s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1904 | 2024-05-29T21:49:57.713Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 633.839307ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1905 | 2024-05-29T21:49:57.861Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 559.766548ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1906 | 2024-05-29T21:49:57.898Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 1.174727946s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 1907 | 2024-05-29T21:49:57.912Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 1.09041071s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1908 | 2024-05-29T21:49:57.950Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 896.291493ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1909 | 2024-05-29T21:49:58.014Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 866.124545ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1910 | 2024-05-29T21:49:58.089Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 1.325453892s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1911 | 2024-05-29T21:49:58.175Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 1.092409648s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1912 | 2024-05-29T21:49:58.359Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 1.039493336s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1913 | 2024-05-29T21:49:58.434Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 1.226082375s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1914 | 2024-05-29T21:49:58.467Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 1.434894068s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1915 | 2024-05-29T21:49:58.707Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 540.00334ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1916 | 2024-05-29T21:49:58.860Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 768.930093ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1917 | 2024-05-29T21:49:58.894Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 1.032492839s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1918 | 2024-05-29T21:49:58.917Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 1.265144989s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1919 | 2024-05-29T21:49:59.015Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 749.204473ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1920 | 2024-05-29T21:49:59.086Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 1.476618972s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 1921 | 2024-05-29T21:49:59.128Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 1.129648122s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1922 | 2024-05-29T21:49:59.181Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 506.563007ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1923 | 2024-05-29T21:49:59.259Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 914.734996ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1924 | 2024-05-29T21:49:59.280Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 1.403361536s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1925 | 2024-05-29T21:49:59.413Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 1.25692645s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1926 | 2024-05-29T21:49:59.427Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 999.213125ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1927 | 2024-05-29T21:49:59.643Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 1.258892687s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1928 | 2024-05-29T21:49:59.674Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 1.284909139s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1929 | 2024-05-29T21:49:59.701Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 1.266663391s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1930 | 2024-05-29T21:49:59.777Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 1.034317633s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1931 | 2024-05-29T21:49:59.913Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 1.398716773s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1932 | 2024-05-29T21:49:59.935Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 763.888331ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1933 | 2024-05-29T21:50:00.183Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 651.066382ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1934 | 2024-05-29T21:50:00.191Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 1.290952128s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1935 | 2024-05-29T21:50:00.266Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 1.352560756s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1936 | 2024-05-29T21:50:00.435Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 885.091008ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1937 | 2024-05-29T21:50:00.571Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 1.304879083s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 1938 | 2024-05-29T21:50:00.679Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 757.919505ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1939 | 2024-05-29T21:50:00.692Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 1.190025055s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1940 | 2024-05-29T21:50:00.707Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 676.384004ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1941 | 2024-05-29T21:50:00.822Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 552.8158ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1942 | 2024-05-29T21:50:00.844Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 663.111101ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1943 | 2024-05-29T21:50:00.911Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 1.356237889s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1944 | 2024-05-29T21:50:00.967Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 1.409873905s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1945 | 2024-05-29T21:50:00.976Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 1.123018181s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1946 | 2024-05-29T21:50:01.321Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 1.356409181s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1947 | 2024-05-29T21:50:01.328Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 540.685663ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1948 | 2024-05-29T21:50:01.384Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 948.88157ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1949 | 2024-05-29T21:50:01.392Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 698.277858ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1950 | 2024-05-29T21:50:01.447Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 670.124505ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1951 | 2024-05-29T21:50:01.491Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 907.458343ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1952 | 2024-05-29T21:50:01.517Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 1.182258612s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1953 | 2024-05-29T21:50:01.628Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 1.258262628s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1954 | 2024-05-29T21:50:01.879Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 874.929043ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1955 | 2024-05-29T21:50:01.887Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 1.245182768s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 1956 | 2024-05-29T21:50:01.894Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 1.099544417s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1957 | 2024-05-29T21:50:02.100Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 1.240360388s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1958 | 2024-05-29T21:50:02.108Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 615.116847ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1959 | 2024-05-29T21:50:02.125Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 591.344085ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1960 | 2024-05-29T21:50:02.275Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 684.512253ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1961 | 2024-05-29T21:50:02.341Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 1.23955431s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1962 | 2024-05-29T21:50:02.386Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 1.096781137s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1963 | 2024-05-29T21:50:02.408Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 963.851214ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1964 | 2024-05-29T21:50:02.686Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 713.871081ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1965 | 2024-05-29T21:50:02.707Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 1.127388021s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1966 | 2024-05-29T21:50:02.726Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 830.080223ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1967 | 2024-05-29T21:50:02.734Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 1.177066602s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1968 | 2024-05-29T21:50:02.763Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 890.999176ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1969 | 2024-05-29T21:50:02.896Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 1.034017201s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1970 | 2024-05-29T21:50:02.968Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 1.059174088s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1971 | 2024-05-29T21:50:03.002Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 1.037935174s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1972 | 2024-05-29T21:50:03.142Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 1.495611131s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 1973 | 2024-05-29T21:50:03.350Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 1.462170384s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1974 | 2024-05-29T21:50:03.380Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 901.104285ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1975 | 2024-05-29T21:50:03.409Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 1.480968436s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1976 | 2024-05-29T21:50:03.491Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 1.119975669s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1977 | 2024-05-29T21:50:03.565Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 705.374551ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1978 | 2024-05-29T21:50:03.591Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 589.879863ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1979 | 2024-05-29T21:50:03.663Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 1.244357661s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1980 | 2024-05-29T21:50:03.844Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 505.44554ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1981 | 2024-05-29T21:50:03.920Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 558.476384ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1982 | 2024-05-29T21:50:03.939Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 526.262877ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1983 | 2024-05-29T21:50:04.037Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 1.190945342s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 1984 | 2024-05-29T21:50:04.049Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 646.122136ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1985 | 2024-05-29T21:50:04.190Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 987.862797ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1986 | 2024-05-29T21:50:04.278Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 508.58572ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1987 | 2024-05-29T21:50:04.289Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 1.076063163s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 1988 | 2024-05-29T21:50:04.359Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 1.045046929s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 1989 | 2024-05-29T21:50:04.473Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 1.319801976s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 1990 | 2024-05-29T21:50:04.487Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 992.566439ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 1991 | 2024-05-29T21:50:04.625Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 720.042056ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 1992 | 2024-05-29T21:50:04.651Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 537.178051ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 1993 | 2024-05-29T21:50:04.712Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 990.117366ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 1994 | 2024-05-29T21:50:04.828Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 1.461875219s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 1995 | 2024-05-29T21:50:04.844Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 1.291071581s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 1996 | 2024-05-29T21:50:04.905Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 911.447661ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 1997 | 2024-05-29T21:50:04.920Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 953.589493ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 1998 | 2024-05-29T21:50:05.194Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 1.439110496s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 1999 | 2024-05-29T21:50:05.206Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 1.394670765s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 2000 | 2024-05-29T21:50:05.245Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 778.269899ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 2001 | 2024-05-29T21:50:05.358Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 1.180345565s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 2002 | 2024-05-29T21:50:05.363Z | INFO | SledAgent (ServiceManager): Profile for oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService10"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::e"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/clickhouse">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::e"/>
<propval type="astring" name="listen_port" value="8123"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:1::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2003 | 2024-05-29T21:50:05.364Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2004 | 2024-05-29T21:50:05.381Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 1.30855667s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 2005 | 2024-05-29T21:50:05.416Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 1.195049344s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 2006 | 2024-05-29T21:50:05.493Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 1.460012504s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 2007 | 2024-05-29T21:50:05.588Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 64.44205ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2008 | 2024-05-29T21:50:05.666Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 133.876624ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2009 | 2024-05-29T21:50:05.726Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 656.836571ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2010 | 2024-05-29T21:50:05.877Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 1.260952861s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 2011 | 2024-05-29T21:50:05.915Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 299.443671ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2012 | 2024-05-29T21:50:05.926Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 1.109373874s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 2013 | 2024-05-29T21:50:05.937Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 547.503122ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 2014 | 2024-05-29T21:50:06.034Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 906.694814ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 2015 | 2024-05-29T21:50:06.145Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 582.452103ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 2016 | 2024-05-29T21:50:06.224Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 538.501511ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2017 | 2024-05-29T21:50:06.299Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 916.799921ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 2018 | 2024-05-29T21:50:06.393Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 1.309286706s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2019 | 2024-05-29T21:50:06.410Z | INFO | SledAgent (ServiceManager): Profile for oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService25"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::8"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte2"/>
<propval type="astring" name="gateway" value="172.30.1.1"/>
<propval type="astring" name="ip" value="172.30.1.5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/external_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:101::8]:5353"/>
<propval type="astring" name="dns_address" value="172.30.1.5:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2020 | 2024-05-29T21:50:06.411Z | INFO | SledAgent (ServiceManager): Profile for oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService28"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::9"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte5"/>
<propval type="astring" name="gateway" value="172.30.1.1"/>
<propval type="astring" name="ip" value="172.30.1.6"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/external_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:101::9]:5353"/>
<propval type="astring" name="dns_address" value="172.30.1.6:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2021 | 2024-05-29T21:50:06.411Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2022 | 2024-05-29T21:50:06.415Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2023 | 2024-05-29T21:50:06.495Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 592.609045ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 2024 | 2024-05-29T21:50:06.549Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 1.499165901s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 2025 | 2024-05-29T21:50:06.563Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 33.454079ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2026 | 2024-05-29T21:50:06.591Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 35.042555ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2027 | 2024-05-29T21:50:06.608Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 139.572027ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2028 | 2024-05-29T21:50:06.618Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 649.03896ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 2029 | 2024-05-29T21:50:06.628Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 1.154074509s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 2030 | 2024-05-29T21:50:06.637Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 79.137083ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2031 | 2024-05-29T21:50:06.647Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 1.307152629s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 2032 | 2024-05-29T21:50:06.706Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 1.26892542s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 2033 | 2024-05-29T21:50:06.740Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 240.802145ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2034 | 2024-05-29T21:50:06.757Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 810.374323ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 2035 | 2024-05-29T21:50:06.770Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 277.397376ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2036 | 2024-05-29T21:50:06.788Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 705.980428ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2037 | 2024-05-29T21:50:06.952Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 537.047761ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 2038 | 2024-05-29T21:50:06.961Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 772.618182ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 2039 | 2024-05-29T21:50:06.993Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 330.459073ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2040 | 2024-05-29T21:50:07.045Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 1.196058248s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 2041 | 2024-05-29T21:50:07.060Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 583.964385ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2042 | 2024-05-29T21:50:07.097Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 1.287016252s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 2043 | 2024-05-29T21:50:07.147Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 1.07284459s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 2044 | 2024-05-29T21:50:07.227Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 961.329322ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 2045 | 2024-05-29T21:50:07.265Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService26"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::b"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte3"/>
<propval type="astring" name="gateway" value="172.30.2.1"/>
<propval type="astring" name="ip" value="172.30.2.6"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2046 | 2024-05-29T21:50:07.265Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2047 | 2024-05-29T21:50:07.271Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService27"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::c"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte4"/>
<propval type="astring" name="gateway" value="172.30.2.1"/>
<propval type="astring" name="ip" value="172.30.2.7"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2048 | 2024-05-29T21:50:07.279Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 1.305534172s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 2049 | 2024-05-29T21:50:07.280Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService24"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::a"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte1"/>
<propval type="astring" name="gateway" value="172.30.2.1"/>
<propval type="astring" name="ip" value="172.30.2.5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 2050 | 2024-05-29T21:50:07.280Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2051 | 2024-05-29T21:50:07.284Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2052 | 2024-05-29T21:50:07.334Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 610.720354ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2053 | 2024-05-29T21:50:07.422Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de") failed: Property not found. retry in 56.470383ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2054 | 2024-05-29T21:50:07.430Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 44.154415ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2055 | 2024-05-29T21:50:07.436Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd") failed: Property not found. retry in 69.496742ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2056 | 2024-05-29T21:50:07.482Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 83.649675ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2057 | 2024-05-29T21:50:07.490Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de") failed: Property not found. retry in 146.945619ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2058 | 2024-05-29T21:50:07.501Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 565.272528ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 2059 | 2024-05-29T21:50:07.510Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 510.026339ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2060 | 2024-05-29T21:50:07.517Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd") failed: Property not found. retry in 88.632435ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2061 | 2024-05-29T21:50:07.574Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 196.755421ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2062 | 2024-05-29T21:50:07.581Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 1.413888487s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 2063 | 2024-05-29T21:50:07.615Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd") failed: Property not found. retry in 269.55334ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2064 | 2024-05-29T21:50:07.648Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de") failed: Property not found. retry in 265.06525ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2065 | 2024-05-29T21:50:07.656Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 1.145568204s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2066 | 2024-05-29T21:50:07.712Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 1.294868556s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2067 | 2024-05-29T21:50:07.744Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 1.179369804s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 2068 | 2024-05-29T21:50:07.782Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 210.533973ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2069 | 2024-05-29T21:50:07.790Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 680.523387ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 2070 | 2024-05-29T21:50:08.377Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd") failed: Property not found. retry in 224.282084ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2071 | 2024-05-29T21:50:08.418Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 778.458508ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2072 | 2024-05-29T21:50:08.461Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 1.235785759s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 2073 | 2024-05-29T21:50:08.481Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de") failed: Property not found. retry in 271.080874ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2074 | 2024-05-29T21:50:08.495Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 737.369298ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2075 | 2024-05-29T21:50:08.509Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 1.298402526s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 2076 | 2024-05-29T21:50:08.528Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 1.328586394s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 2077 | 2024-05-29T21:50:08.559Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 535.619297ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 2078 | 2024-05-29T21:50:08.571Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 1.125278897s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 2079 | 2024-05-29T21:50:08.610Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 586.677143ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 2080 | 2024-05-29T21:50:08.645Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 1.02351298s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 2081 | 2024-05-29T21:50:08.669Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 1.322425217s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2082 | 2024-05-29T21:50:08.689Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 941.611165ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 2083 | 2024-05-29T21:50:08.698Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 1.166913271s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 2084 | 2024-05-29T21:50:08.726Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd") failed: Property not found. retry in 1.030661245s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2085 | 2024-05-29T21:50:08.737Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 792.827111ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 2086 | 2024-05-29T21:50:08.770Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de") failed: Property not found. retry in 1.000830054s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2087 | 2024-05-29T21:50:08.814Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 1.216848533s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2088 | 2024-05-29T21:50:08.934Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 843.986268ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 2089 | 2024-05-29T21:50:09.006Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 742.931794ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 2090 | 2024-05-29T21:50:09.017Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 689.12011ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2091 | 2024-05-29T21:50:09.103Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 1.392471332s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 2092 | 2024-05-29T21:50:09.206Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 1.276542378s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2093 | 2024-05-29T21:50:09.214Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 741.351769ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 2094 | 2024-05-29T21:50:09.242Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 994.569332ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2095 | 2024-05-29T21:50:09.540Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 1.25804297s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 2096 | 2024-05-29T21:50:09.641Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 1.480523155s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 2097 | 2024-05-29T21:50:09.678Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 1.339986562s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 2098 | 2024-05-29T21:50:09.704Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 1.148781929s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 2099 | 2024-05-29T21:50:09.712Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 1.477330486s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 2100 | 2024-05-29T21:50:09.720Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 1.147499739s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2101 | 2024-05-29T21:50:09.759Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 1.215217296s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 2102 | 2024-05-29T21:50:09.768Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd") failed: Property not found. retry in 1.330418153s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2103 | 2024-05-29T21:50:09.781Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de") failed: Property not found. retry in 1.157768714s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2104 | 2024-05-29T21:50:09.788Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 1.357486703s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 2105 | 2024-05-29T21:50:09.816Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 1.35396401s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 2106 | 2024-05-29T21:50:09.867Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 671.691679ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 2107 | 2024-05-29T21:50:09.875Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 739.511368ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 2108 | 2024-05-29T21:50:09.965Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 1.048632429s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 2109 | 2024-05-29T21:50:10.001Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 879.628902ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2110 | 2024-05-29T21:50:10.041Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 953.684963ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2111 | 2024-05-29T21:50:10.248Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 918.393477ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2112 | 2024-05-29T21:50:10.493Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 790.362995ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2113 | 2024-05-29T21:50:10.504Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 858.857811ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 2114 | 2024-05-29T21:50:10.550Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 715.045899ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 2115 | 2024-05-29T21:50:10.628Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 602.743877ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 2116 | 2024-05-29T21:50:10.809Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 896.672422ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 2117 | 2024-05-29T21:50:10.863Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 1.055999594s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 2118 | 2024-05-29T21:50:10.876Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 630.836722ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2119 | 2024-05-29T21:50:10.891Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 926.812195ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2120 | 2024-05-29T21:50:10.949Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de") failed: Property not found. retry in 1.248614724s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2121 | 2024-05-29T21:50:10.987Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 661.652648ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 2122 | 2024-05-29T21:50:11.006Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 744.475393ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2123 | 2024-05-29T21:50:11.022Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 1.420121948s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 2124 | 2024-05-29T21:50:11.031Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 754.767894ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 2125 | 2024-05-29T21:50:11.108Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd") failed: Property not found. retry in 784.672705ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2126 | 2024-05-29T21:50:11.131Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 977.330799ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 2127 | 2024-05-29T21:50:11.154Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 1.389295087s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 2128 | 2024-05-29T21:50:11.179Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 673.29273ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2129 | 2024-05-29T21:50:11.191Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 1.00908859s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 2130 | 2024-05-29T21:50:11.202Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 1.253844894s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 2131 | 2024-05-29T21:50:11.243Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 961.336078ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 2132 | 2024-05-29T21:50:11.277Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 1.172178632s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 2133 | 2024-05-29T21:50:11.293Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 583.514289ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2134 | 2024-05-29T21:50:11.377Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 844.313354ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 2135 | 2024-05-29T21:50:11.518Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 534.500811ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2136 | 2024-05-29T21:50:11.663Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 661.886064ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 2137 | 2024-05-29T21:50:11.718Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 1.2950414s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 2138 | 2024-05-29T21:50:11.764Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 664.824786ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2139 | 2024-05-29T21:50:11.798Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 1.07948791s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 2140 | 2024-05-29T21:50:11.831Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 1.458487491s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2141 | 2024-05-29T21:50:11.866Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 507.327237ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2142 | 2024-05-29T21:50:11.889Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 954.706118ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2143 | 2024-05-29T21:50:11.904Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd") failed: Property not found. retry in 714.567885ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2144 | 2024-05-29T21:50:11.934Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 626.655679ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 2145 | 2024-05-29T21:50:11.965Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:52840
|
| 2146 | 2024-05-29T21:50:11.969Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 2580
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:52840
req_id = 5f79234c-d84b-4e14-a375-3085bfd768e7
response_code = 200
uri = /baseboard
|
| 2147 | 2024-05-29T21:50:12.074Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 911.043998ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2148 | 2024-05-29T21:50:12.126Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 515.997326ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 2149 | 2024-05-29T21:50:12.216Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de") failed: Property not found. retry in 1.096926479s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2150 | 2024-05-29T21:50:12.233Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 530.915311ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 2151 | 2024-05-29T21:50:12.251Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 976.960757ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 2152 | 2024-05-29T21:50:12.270Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 615.460829ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 2153 | 2024-05-29T21:50:12.340Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 979.27649ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 2154 | 2024-05-29T21:50:12.390Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 1.363176362s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2155 | 2024-05-29T21:50:12.447Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 608.813696ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2156 | 2024-05-29T21:50:12.464Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 1.156047494s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 2157 | 2024-05-29T21:50:12.478Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 692.973677ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 2158 | 2024-05-29T21:50:12.491Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 1.326143032s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 2159 | 2024-05-29T21:50:12.557Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 891.364711ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 2160 | 2024-05-29T21:50:12.573Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 1.304266277s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 2161 | 2024-05-29T21:50:12.639Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd") failed: Property not found. retry in 505.17901ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2162 | 2024-05-29T21:50:12.673Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 728.131852ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 2163 | 2024-05-29T21:50:12.788Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 807.972704ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 2164 | 2024-05-29T21:50:12.858Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 745.321244ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2165 | 2024-05-29T21:50:12.899Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 1.270230878s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 2166 | 2024-05-29T21:50:12.913Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 501.888109ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 2167 | 2024-05-29T21:50:13.003Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 834.081161ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2168 | 2024-05-29T21:50:13.028Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 1.37956294s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 2169 | 2024-05-29T21:50:13.069Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 658.218015ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2170 | 2024-05-29T21:50:13.161Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd") failed: Property not found. retry in 1.253757612s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2171 | 2024-05-29T21:50:13.188Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 694.16057ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 2172 | 2024-05-29T21:50:13.244Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 782.255841ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 2173 | 2024-05-29T21:50:13.306Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 1.003055512s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2174 | 2024-05-29T21:50:13.333Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de") failed: Property not found. retry in 1.077063587s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2175 | 2024-05-29T21:50:13.353Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 1.184438993s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 2176 | 2024-05-29T21:50:13.415Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 1.107285086s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 2177 | 2024-05-29T21:50:13.429Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 1.250125572s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 2178 | 2024-05-29T21:50:13.460Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 754.563077ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 2179 | 2024-05-29T21:50:13.610Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 1.314426907s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 2180 | 2024-05-29T21:50:13.621Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 598.698881ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2181 | 2024-05-29T21:50:13.632Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 1.063423621s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 2182 | 2024-05-29T21:50:13.741Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 1.196460921s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2183 | 2024-05-29T21:50:13.767Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 1.113846907s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2184 | 2024-05-29T21:50:13.828Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 1.408782399s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 2185 | 2024-05-29T21:50:13.846Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 508.801604ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2186 | 2024-05-29T21:50:13.888Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 871.964645ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 2187 | 2024-05-29T21:50:13.899Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 871.417408ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 2188 | 2024-05-29T21:50:14.044Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 971.362669ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 2189 | 2024-05-29T21:50:14.182Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 1.463459115s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 2190 | 2024-05-29T21:50:14.224Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 527.499661ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 2191 | 2024-05-29T21:50:14.235Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 1.132283159s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2192 | 2024-05-29T21:50:14.322Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 914.852929ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2193 | 2024-05-29T21:50:14.366Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 896.898001ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2194 | 2024-05-29T21:50:14.419Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 1.229863038s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 2195 | 2024-05-29T21:50:14.429Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd") failed: Property not found. retry in 1.462325498s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2196 | 2024-05-29T21:50:14.442Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de") failed: Property not found. retry in 1.120477536s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2197 | 2024-05-29T21:50:14.537Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 1.398921447s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 2198 | 2024-05-29T21:50:14.551Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 1.192119938s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 2199 | 2024-05-29T21:50:14.695Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 686.833412ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 2200 | 2024-05-29T21:50:14.709Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 683.090227ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 2201 | 2024-05-29T21:50:14.763Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 1.049032758s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 2202 | 2024-05-29T21:50:14.775Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 537.785943ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 2203 | 2024-05-29T21:50:14.785Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 724.952479ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 2204 | 2024-05-29T21:50:14.892Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 824.137199ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2205 | 2024-05-29T21:50:14.938Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 739.248451ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 2206 | 2024-05-29T21:50:14.952Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 1.2636599s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2207 | 2024-05-29T21:50:15.029Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 1.254280979s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 2208 | 2024-05-29T21:50:15.252Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 1.290272049s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 2209 | 2024-05-29T21:50:15.264Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 874.078296ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2210 | 2024-05-29T21:50:15.274Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 893.038127ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2211 | 2024-05-29T21:50:15.325Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 794.426566ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 2212 | 2024-05-29T21:50:15.383Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 1.120076635s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2213 | 2024-05-29T21:50:15.395Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 1.298239654s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 2214 | 2024-05-29T21:50:15.407Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 571.464715ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 2215 | 2024-05-29T21:50:15.522Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 866.035591ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 2216 | 2024-05-29T21:50:15.579Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de") failed: Property not found. retry in 719.339772ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2217 | 2024-05-29T21:50:15.662Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 673.048066ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 2218 | 2024-05-29T21:50:15.676Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 1.467173326s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 2219 | 2024-05-29T21:50:15.690Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 708.475383ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 2220 | 2024-05-29T21:50:15.733Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 1.084323777s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2221 | 2024-05-29T21:50:15.759Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 1.24909442s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 2222 | 2024-05-29T21:50:15.829Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 942.434329ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 2223 | 2024-05-29T21:50:15.907Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd") failed: Property not found. retry in 913.732983ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2224 | 2024-05-29T21:50:15.948Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 701.504683ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 2225 | 2024-05-29T21:50:15.992Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 1.346591094s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 2226 | 2024-05-29T21:50:16.135Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 687.868483ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 2227 | 2024-05-29T21:50:16.153Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 662.911901ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2228 | 2024-05-29T21:50:16.183Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 1.475800209s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2229 | 2024-05-29T21:50:16.228Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 1.016118362s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2230 | 2024-05-29T21:50:16.295Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c") failed: Property not found. retry in 1.430164507s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 2231 | 2024-05-29T21:50:16.316Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de") failed: Property not found. retry in 857.114247ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2232 | 2024-05-29T21:50:16.352Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 1.006489172s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 2233 | 2024-05-29T21:50:16.404Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 802.422864ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 2234 | 2024-05-29T21:50:16.419Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 740.038956ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 2235 | 2024-05-29T21:50:16.515Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 582.889734ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2236 | 2024-05-29T21:50:16.553Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80") failed: Property not found. retry in 971.527612ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 2237 | 2024-05-29T21:50:16.662Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af") failed: Property not found. retry in 1.266633042s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 2238 | 2024-05-29T21:50:16.704Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 863.453732ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 2239 | 2024-05-29T21:50:16.782Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b") failed: Property not found. retry in 1.128789596s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 2240 | 2024-05-29T21:50:16.827Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 617.780874ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2241 | 2024-05-29T21:50:16.839Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd") failed: Property not found. retry in 1.068532506s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2242 | 2024-05-29T21:50:16.849Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 660.818465ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 2243 | 2024-05-29T21:50:16.858Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 1.071089186s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2244 | 2024-05-29T21:50:17.020Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5") failed: Property not found. retry in 927.426116ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 2245 | 2024-05-29T21:50:17.108Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 763.196137ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2246 | 2024-05-29T21:50:17.152Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50") failed: Property not found. retry in 813.959022ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 2247 | 2024-05-29T21:50:17.167Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027") failed: Property not found. retry in 1.238207054s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 2248 | 2024-05-29T21:50:17.181Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de") failed: Property not found. retry in 932.207251ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2249 | 2024-05-29T21:50:17.215Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848") failed: Property not found. retry in 948.984542ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 2250 | 2024-05-29T21:50:17.255Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 598.526683ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2251 | 2024-05-29T21:50:17.350Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8") failed: Property not found. retry in 1.487697013s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 2252 | 2024-05-29T21:50:17.368Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2") failed: Property not found. retry in 1.200530782s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 2253 | 2024-05-29T21:50:17.456Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7") failed: Property not found. retry in 1.360849256s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2254 | 2024-05-29T21:50:17.523Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66") failed: Property not found. retry in 931.311983ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 2255 | 2024-05-29T21:50:17.584Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765") failed: Property not found. retry in 1.207682176s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 2256 | 2024-05-29T21:50:17.675Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9") failed: Property not found. retry in 891.631766ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2257 | 2024-05-29T21:50:17.869Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5") failed: Property not found. retry in 1.212554366s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2258 | 2024-05-29T21:50:17.900Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110") failed: Property not found. retry in 1.07079292s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2259 | 2024-05-29T21:50:17.925Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd") failed: Property not found. retry in 1.308131069s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2260 | 2024-05-29T21:50:17.956Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c") failed: Property not found. retry in 1.28553234s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2261 | 2024-05-29T21:50:18.129Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de") failed: Property not found. retry in 647.927514ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2262 | 2024-05-29T21:50:19.480Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd
|
| 2263 | 2024-05-29T21:50:19.480Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110
|
| 2264 | 2024-05-29T21:50:19.480Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66
|
| 2265 | 2024-05-29T21:50:19.480Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5
|
| 2266 | 2024-05-29T21:50:19.480Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de
|
| 2267 | 2024-05-29T21:50:19.480Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c
|
| 2268 | 2024-05-29T21:50:19.480Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027
|
| 2269 | 2024-05-29T21:50:19.480Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848
|
| 2270 | 2024-05-29T21:50:19.481Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9
|
| 2271 | 2024-05-29T21:50:19.481Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50
|
| 2272 | 2024-05-29T21:50:19.481Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765
|
| 2273 | 2024-05-29T21:50:19.481Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b
|
| 2274 | 2024-05-29T21:50:19.481Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80
|
| 2275 | 2024-05-29T21:50:19.481Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8
|
| 2276 | 2024-05-29T21:50:19.481Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2
|
| 2277 | 2024-05-29T21:50:19.481Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7
|
| 2278 | 2024-05-29T21:50:19.481Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af
|
| 2279 | 2024-05-29T21:50:19.481Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5
|
| 2280 | 2024-05-29T21:50:19.481Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c
|
| 2281 | 2024-05-29T21:50:19.492Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 2282 | 2024-05-29T21:50:19.501Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 2283 | 2024-05-29T21:50:19.514Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 37703488
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:47072
req_id = 170e80e8-b1ba-467b-9900-88068a554b84
response_code = 204
uri = /omicron-zones
|
| 2284 | 2024-05-29T21:50:19.538Z | INFO | SledAgent (RSS): Finished setting up services
file = sled-agent/src/rack_setup/service.rs:1236
|
| 2285 | 2024-05-29T21:50:19.538Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.rss-plan-completed.marker.tmp
file = common/src/ledger.rs:199
|
| 2286 | 2024-05-29T21:50:19.541Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.rss-plan-completed.marker.tmp
file = common/src/ledger.rs:199
|
| 2287 | 2024-05-29T21:50:19.566Z | INFO | SledAgent (RSS): Handing off control to Nexus
file = sled-agent/src/rack_setup/service.rs:680
|
| 2288 | 2024-05-29T21:50:19.570Z | INFO | SledAgent (RSS): Nexus address: [fd00:1122:3344:101::a]:12221
file = sled-agent/src/rack_setup/service.rs:693
|
| 2289 | 2024-05-29T21:50:19.648Z | INFO | SledAgent (RSS): rack_network_config: RackNetworkConfigV1 {
bfd: [],
bgp: [],
infra_ip_first: 10.151.2.174,
infra_ip_last: 10.151.2.174,
ports: [
PortConfigV1 {
addresses: [
V4(
Ipv4Network {
addr: 10.151.2.174,
prefix: 24,
},
),
],
autoneg: false,
bgp_peers: [],
port: "qsfp0",
routes: [
RouteConfig {
destination: V4(
Ipv4Network {
addr: 0.0.0.0,
prefix: 0,
},
),
nexthop: 10.151.2.100,
vlan_id: None,
},
],
switch: Switch0,
uplink_port_fec: None,
uplink_port_speed: Speed40G,
},
],
rack_subnet: Ipv6Network {
addr: fd00:1122:3344:100::,
prefix: 56,
},
}
file = sled-agent/src/rack_setup/service.rs:817
|
| 2290 | 2024-05-29T21:50:20.795Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2291 | 2024-05-29T21:50:20.919Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2292 | 2024-05-29T21:50:21.004Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2293 | 2024-05-29T21:50:21.211Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2294 | 2024-05-29T21:50:21.455Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2295 | 2024-05-29T21:50:21.716Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2296 | 2024-05-29T21:50:22.076Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2297 | 2024-05-29T21:50:22.440Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2298 | 2024-05-29T21:50:22.770Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2299 | 2024-05-29T21:50:23.167Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2300 | 2024-05-29T21:50:23.658Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2301 | 2024-05-29T21:50:23.787Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 61.080820397s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:417
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2302 | 2024-05-29T21:50:24.309Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2303 | 2024-05-29T21:50:24.783Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2304 | 2024-05-29T21:50:25.506Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2305 | 2024-05-29T21:50:25.695Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/nexus.rs:619
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2306 | 2024-05-29T21:50:26.057Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2307 | 2024-05-29T21:50:26.514Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2308 | 2024-05-29T21:50:26.721Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/nexus.rs:619
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2309 | 2024-05-29T21:50:26.741Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2310 | 2024-05-29T21:50:27.516Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2311 | 2024-05-29T21:50:28.331Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
| 2312 | 2024-05-29T21:50:28.519Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2313 | 2024-05-29T21:50:28.720Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/nexus.rs:619
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2314 | 2024-05-29T21:50:29.522Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/probe_manager.rs:169
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2315 | 2024-05-29T21:50:30.339Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "583d01a1-104f-4949-bbb8-a0165d41021a", "content-length": "173", "date": "Wed, 29 May 2024 21:50:30 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: rack with id \"9090f606-344e-4bff-9fcd-4f77886b6762\"", request_id: "583d01a1-104f-4949-bbb8-a0165d41021a" }
file = sled-agent/src/rack_setup/service.rs:879
|
| 2316 | 2024-05-29T21:50:32.768Z | INFO | SledAgent (NexusNotifierTask): Successfully put SledAgentInfo to nexus
file = sled-agent/src/nexus.rs:592
new_generation = 1
old_generation = 1
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2317 | 2024-05-29T21:50:32.890Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:37490
|
| 2318 | 2024-05-29T21:50:32.891Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2319 | 2024-05-29T21:50:32.975Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 84959
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:37490
req_id = 77027e35-e43f-45c4-aa9d-daccbc793088
response_code = 200
uri = /inventory
|
| 2320 | 2024-05-29T21:50:32.976Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2321 | 2024-05-29T21:50:32.976Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2322 | 2024-05-29T21:50:32.979Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2323 | 2024-05-29T21:50:32.979Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 3522
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:37490
req_id = fa147788-895d-4695-8c48-04fdb2890476
response_code = 200
uri = /omicron-zones
|
| 2324 | 2024-05-29T21:50:33.988Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:53256
|
| 2325 | 2024-05-29T21:50:33.990Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name("external-dns-ce802882-5f60-4fa0-8546-381dc464eec5"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name("external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name("nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name("nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name("nexus-73fd9e9c-3929-404c-966e-fc226ac774de"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2326 | 2024-05-29T21:50:33.990Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2327 | 2024-05-29T21:50:33.991Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2328 | 2024-05-29T21:50:33.991Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2329 | 2024-05-29T21:50:33.991Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2330 | 2024-05-29T21:50:33.991Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2331 | 2024-05-29T21:50:33.991Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2332 | 2024-05-29T21:50:33.991Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 2412
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:53256
req_id = dbe931f6-7484-4252-bf85-1d8f2aa13ced
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2333 | 2024-05-29T21:50:33.993Z | INFO | SledAgent (RSS): Handoff to Nexus is complete
file = sled-agent/src/rack_setup/service.rs:889
|
| 2334 | 2024-05-29T21:50:34.120Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:63978
|
| 2335 | 2024-05-29T21:50:34.120Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2336 | 2024-05-29T21:50:34.204Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 84109
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:63978
req_id = 21592c1e-5aa4-42a3-ab17-7ce5d7f97023
response_code = 200
uri = /inventory
|
| 2337 | 2024-05-29T21:50:34.204Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2338 | 2024-05-29T21:50:34.204Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2339 | 2024-05-29T21:50:34.204Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 2340 | 2024-05-29T21:50:34.205Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 639
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:63978
req_id = 07d91b4d-6853-4545-9546-9ae10e52d50f
response_code = 200
uri = /omicron-zones
|
| 2341 | 2024-05-29T21:50:34.334Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:37270
|
| 2342 | 2024-05-29T21:50:34.334Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name("external-dns-ce802882-5f60-4fa0-8546-381dc464eec5"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name("external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name("nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name("nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name("nexus-73fd9e9c-3929-404c-966e-fc226ac774de"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2343 | 2024-05-29T21:50:34.334Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2344 | 2024-05-29T21:50:34.334Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2345 | 2024-05-29T21:50:34.334Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2346 | 2024-05-29T21:50:34.334Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2347 | 2024-05-29T21:50:34.334Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2348 | 2024-05-29T21:50:34.334Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2349 | 2024-05-29T21:50:34.334Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 344
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:37270
req_id = 3af49ac4-046f-4070-93e2-16b77b666f05
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2350 | 2024-05-29T21:50:34.387Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:40524
|
| 2351 | 2024-05-29T21:50:34.388Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name("external-dns-ce802882-5f60-4fa0-8546-381dc464eec5"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name("external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name("nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name("nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name("nexus-73fd9e9c-3929-404c-966e-fc226ac774de"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2352 | 2024-05-29T21:50:34.388Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2353 | 2024-05-29T21:50:34.388Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2354 | 2024-05-29T21:50:34.388Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2355 | 2024-05-29T21:50:34.388Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2356 | 2024-05-29T21:50:34.388Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2357 | 2024-05-29T21:50:34.388Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2358 | 2024-05-29T21:50:34.388Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 384
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:40524
req_id = 8ece6b47-beeb-47df-a81b-9004c77e1814
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2359 | 2024-05-29T21:50:34.413Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:40189
|
| 2360 | 2024-05-29T21:50:34.414Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name("external-dns-ce802882-5f60-4fa0-8546-381dc464eec5"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name("external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name("nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name("nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name("nexus-73fd9e9c-3929-404c-966e-fc226ac774de"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2361 | 2024-05-29T21:50:34.414Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2362 | 2024-05-29T21:50:34.414Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2363 | 2024-05-29T21:50:34.414Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2364 | 2024-05-29T21:50:34.414Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2365 | 2024-05-29T21:50:34.415Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2366 | 2024-05-29T21:50:34.415Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2367 | 2024-05-29T21:50:34.415Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 1057
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:40189
req_id = d6dd36d1-37e8-4c09-b3dc-fc81b03b7195
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2368 | 2024-05-29T21:50:42.011Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:48490
|
| 2369 | 2024-05-29T21:50:42.012Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 43
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:48490
req_id = 31933dbd-c532-47a4-9057-534be38bbf30
response_code = 200
uri = /baseboard
|
| 2370 | 2024-05-29T21:51:00.217Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:58925
|
| 2371 | 2024-05-29T21:51:00.303Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 86416
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:58925
req_id = ea2038f9-c5fb-442e-8b97-370b25f0412b
response_code = 204
uri = /switch-ports
|
| 2372 | 2024-05-29T21:51:00.303Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:49429
|
| 2373 | 2024-05-29T21:51:00.357Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 53904
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:49429
req_id = 37d946f0-9920-4373-9ef5-ea9b5f545f07
response_code = 204
uri = /switch-ports
|
| 2374 | 2024-05-29T21:51:00.358Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 731
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:58925
req_id = 43107d71-0698-473b-8107-9ac5f1705cfb
response_code = 200
uri = /network-bootstore-config
|
| 2375 | 2024-05-29T21:51:00.359Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 776
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:49429
req_id = 1cce8991-505f-4159-9909-76f15c009fa2
response_code = 200
uri = /network-bootstore-config
|
| 2376 | 2024-05-29T21:51:00.377Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:34154
|
| 2377 | 2024-05-29T21:51:00.424Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 46916
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:34154
req_id = 4090d39d-835d-4a66-af36-4ab406773bb4
response_code = 204
uri = /switch-ports
|
| 2378 | 2024-05-29T21:51:00.424Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 230
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:34154
req_id = 2cf03956-107d-4092-9e29-eb231e12950d
response_code = 200
uri = /network-bootstore-config
|
| 2379 | 2024-05-29T21:51:00.425Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 3, current_generation: 1
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
| 2380 | 2024-05-29T21:51:00.425Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 2381 | 2024-05-29T21:51:00.427Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 2382 | 2024-05-29T21:51:00.428Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 3
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-sock-i86pc
|
| 2383 | 2024-05-29T21:51:00.428Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 2, current_generation: 3
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
| 2384 | 2024-05-29T21:51:00.428Z | ERRO | SledAgent (bootstore): Attempted network config update with stale generation: attemped_update_generation: 2, current_generation: 3
file = bootstore/src/schemes/v0/peer.rs:574
peer_id = pc-sock-i86pc
|
| 2385 | 2024-05-29T21:51:00.428Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 4234
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:49429
req_id = 6f574fed-560c-49e9-9fef-ae9536b4d7aa
response_code = 204
uri = /network-bootstore-config
|
| 2386 | 2024-05-29T21:51:00.428Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = failed to write updated config to boot store: Network config update failed because it is out of date. Attempted update generation: 2, current generation: 3
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866
latency_us = 4110
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:58925
req_id = 175cafd9-f8b7-44be-9dd3-77317f88ef35
response_code = 500
uri = /network-bootstore-config
|
| 2387 | 2024-05-29T21:51:00.434Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 4, current_generation: 3
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
| 2388 | 2024-05-29T21:51:00.434Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 2389 | 2024-05-29T21:51:00.435Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 2390 | 2024-05-29T21:51:00.435Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 4
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-sock-i86pc
|
| 2391 | 2024-05-29T21:51:00.435Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 630
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:34154
req_id = c60f8768-8909-43ca-b0c2-122ae46b4826
response_code = 204
uri = /network-bootstore-config
|
| 2392 | 2024-05-29T21:51:12.045Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:50294
|
| 2393 | 2024-05-29T21:51:12.045Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 77
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:50294
req_id = 20a306f1-41e3-45e3-ba32-15db4938a768
response_code = 200
uri = /baseboard
|
| 2394 | 2024-05-29T21:51:30.480Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:58888
|
| 2395 | 2024-05-29T21:51:30.526Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 46193
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:58888
req_id = 153c99ff-3446-4078-87ee-46ad5de8ab83
response_code = 204
uri = /switch-ports
|
| 2396 | 2024-05-29T21:51:30.526Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:36531
|
| 2397 | 2024-05-29T21:51:30.533Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 6593
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:58888
req_id = 4facb570-8743-4213-a76f-a9028aa5aa47
response_code = 200
uri = /network-bootstore-config
|
| 2398 | 2024-05-29T21:51:30.579Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 52940
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:36531
req_id = 5d71627b-22af-4b15-9c91-650f07a5324d
response_code = 204
uri = /switch-ports
|
| 2399 | 2024-05-29T21:51:30.580Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 150
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:36531
req_id = adfbfe18-e315-44c0-8d61-15d175b4be28
response_code = 200
uri = /network-bootstore-config
|
| 2400 | 2024-05-29T21:51:30.688Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:61956
|
| 2401 | 2024-05-29T21:51:30.729Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 40780
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:61956
req_id = cce4446e-b321-4605-b402-5f9afecf6585
response_code = 204
uri = /switch-ports
|
| 2402 | 2024-05-29T21:51:30.729Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 104
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:61956
req_id = e8d1cec6-9bd3-4462-8c09-3653624be56a
response_code = 200
uri = /network-bootstore-config
|
| 2403 | 2024-05-29T21:51:42.083Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:46268
|
| 2404 | 2024-05-29T21:51:42.083Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 70
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:46268
req_id = af3af33b-eb6a-487a-81fe-8430256ee3ed
response_code = 200
uri = /baseboard
|
| 2405 | 2024-05-29T21:51:54.998Z | INFO | SledAgent (dropshot): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:61502
remote_addr = [fd00:1122:3344:101::d]:45465
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2406 | 2024-05-29T21:51:55.009Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 10543
local_addr = [fd00:1122:3344:101::1]:61502
method = GET
remote_addr = [fd00:1122:3344:101::d]:45465
req_id = 4baa1524-2580-4a92-ba1b-479d71ee435c
response_code = 200
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
uri = /67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2407 | 2024-05-29T21:51:58.888Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
| 2408 | 2024-05-29T21:52:00.233Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:40833
|
| 2409 | 2024-05-29T21:52:00.280Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 46495
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:40833
req_id = a47b86ca-5bfa-4cff-a3a9-3c189c3b61b8
response_code = 204
uri = /switch-ports
|
| 2410 | 2024-05-29T21:52:00.280Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:48347
|
| 2411 | 2024-05-29T21:52:00.280Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:48089
|
| 2412 | 2024-05-29T21:52:00.284Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 3610
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:40833
req_id = dd599def-40b3-4771-b3dc-4f89dce0feec
response_code = 200
uri = /network-bootstore-config
|
| 2413 | 2024-05-29T21:52:00.323Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 43015
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:48347
req_id = 09a15d67-50bb-4443-8b29-94d42d3cabef
response_code = 204
uri = /switch-ports
|
| 2414 | 2024-05-29T21:52:00.327Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 582
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:48347
req_id = 5491d6d6-23f1-41c2-9b03-986f6d60e05c
response_code = 200
uri = /network-bootstore-config
|
| 2415 | 2024-05-29T21:52:00.368Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 87325
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:48089
req_id = 4702a047-9a32-4bae-a8fe-5ef65b1acd5e
response_code = 204
uri = /switch-ports
|
| 2416 | 2024-05-29T21:52:00.368Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 86
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:48089
req_id = deae6334-ec43-40de-8528-883f1ba95f5d
response_code = 200
uri = /network-bootstore-config
|
| 2417 | 2024-05-29T21:52:12.116Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:55474
|
| 2418 | 2024-05-29T21:52:12.116Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 48
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:55474
req_id = 950c4367-0db6-4c3d-8b43-87437840916a
response_code = 200
uri = /baseboard
|
| 2419 | 2024-05-29T21:52:24.999Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 1088
local_addr = [fd00:1122:3344:101::1]:61502
method = GET
remote_addr = [fd00:1122:3344:101::d]:45465
req_id = 4433369a-4b12-45ec-a1b2-d9c95cfb678b
response_code = 200
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
uri = /67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2420 | 2024-05-29T21:52:30.423Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:44946
|
| 2421 | 2024-05-29T21:52:30.470Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 46800
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:44946
req_id = 9d3b842e-17a3-4568-ab7c-ee5f4fcd9b63
response_code = 204
uri = /switch-ports
|
| 2422 | 2024-05-29T21:52:30.470Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:64010
|
| 2423 | 2024-05-29T21:52:30.514Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 43619
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:64010
req_id = 5b6bf734-a5d5-45f8-b53a-365b814330a4
response_code = 204
uri = /switch-ports
|
| 2424 | 2024-05-29T21:52:30.514Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 97
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:44946
req_id = 4b2b5451-9313-4016-96a2-ff57f6fbf230
response_code = 200
uri = /network-bootstore-config
|
| 2425 | 2024-05-29T21:52:30.514Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 117
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:64010
req_id = 0b451e73-28f7-48aa-b68b-f95f4dc3b8f7
response_code = 200
uri = /network-bootstore-config
|
| 2426 | 2024-05-29T21:52:30.795Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:64316
|
| 2427 | 2024-05-29T21:52:30.838Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 42961
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:64316
req_id = 29f83a05-6e7e-426c-a08d-3a6d5ac7ce52
response_code = 204
uri = /switch-ports
|
| 2428 | 2024-05-29T21:52:30.838Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 99
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:64316
req_id = 74182a54-07a2-441c-b348-e02761827820
response_code = 200
uri = /network-bootstore-config
|
| 2429 | 2024-05-29T21:52:42.147Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:54545
|
| 2430 | 2024-05-29T21:52:42.147Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 59
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:54545
req_id = ec096f27-1966-4a18-8ef1-0b4d70d27328
response_code = 200
uri = /baseboard
|
| 2431 | 2024-05-29T21:52:54.998Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 443
local_addr = [fd00:1122:3344:101::1]:61502
method = GET
remote_addr = [fd00:1122:3344:101::d]:45465
req_id = 104ceac0-3923-4668-882d-fa8457446f2a
response_code = 200
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
uri = /67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2432 | 2024-05-29T21:53:00.174Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:40807
|
| 2433 | 2024-05-29T21:53:00.221Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 47522
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:40807
req_id = b0c575f1-ecea-4d7c-ba2a-ae896e1998e0
response_code = 204
uri = /switch-ports
|
| 2434 | 2024-05-29T21:53:00.222Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 119
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:40807
req_id = 684eb870-8db5-4a69-87df-d92823db3baa
response_code = 200
uri = /network-bootstore-config
|
| 2435 | 2024-05-29T21:53:00.367Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:38611
|
| 2436 | 2024-05-29T21:53:00.414Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 46806
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:38611
req_id = a12c8620-ff4f-4480-a418-dfb4c8270352
response_code = 204
uri = /switch-ports
|
| 2437 | 2024-05-29T21:53:00.414Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:36139
|
| 2438 | 2024-05-29T21:53:00.418Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 3082
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:38611
req_id = 014d4fe7-6cbf-485d-8a63-20854da810a1
response_code = 200
uri = /network-bootstore-config
|
| 2439 | 2024-05-29T21:53:00.457Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 42291
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:36139
req_id = a3267520-8b18-45ce-b43c-207680f4ca88
response_code = 204
uri = /switch-ports
|
| 2440 | 2024-05-29T21:53:00.457Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 150
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:36139
req_id = 6d853065-65e5-4f31-b609-f737000a6bea
response_code = 200
uri = /network-bootstore-config
|
| 2441 | 2024-05-29T21:53:12.179Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:51467
|
| 2442 | 2024-05-29T21:53:12.179Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 66
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:51467
req_id = 04ee2c1e-9553-4a2f-a245-798ce095699b
response_code = 200
uri = /baseboard
|
| 2443 | 2024-05-29T21:53:24.997Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 732
local_addr = [fd00:1122:3344:101::1]:61502
method = GET
remote_addr = [fd00:1122:3344:101::d]:45465
req_id = a4eab00b-a471-48b9-912d-0f3e3711aaa8
response_code = 200
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
uri = /67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2444 | 2024-05-29T21:53:30.487Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:39781
|
| 2445 | 2024-05-29T21:53:30.532Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 45169
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:39781
req_id = 74ae45cb-d9e2-4881-9464-461dc5f28176
response_code = 204
uri = /switch-ports
|
| 2446 | 2024-05-29T21:53:30.532Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:44935
|
| 2447 | 2024-05-29T21:53:30.535Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 3112
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:39781
req_id = 48c8c2fb-d8cd-4bd9-8918-99534047d04e
response_code = 200
uri = /network-bootstore-config
|
| 2448 | 2024-05-29T21:53:30.575Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 42270
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:44935
req_id = 4fc6703b-4d49-4969-9647-c4afd71fbcb3
response_code = 204
uri = /switch-ports
|
| 2449 | 2024-05-29T21:53:30.575Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 162
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:44935
req_id = f0541c5a-9bc1-440d-9f34-4660b012c8fa
response_code = 200
uri = /network-bootstore-config
|
| 2450 | 2024-05-29T21:53:30.714Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:63575
|
| 2451 | 2024-05-29T21:53:30.758Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 44115
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:63575
req_id = 6fd650a6-0fca-4c4b-84c5-6c448d8a2ceb
response_code = 204
uri = /switch-ports
|
| 2452 | 2024-05-29T21:53:30.759Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 603
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:63575
req_id = 8fe5ad5f-b6ff-4460-a8cd-f35f502a8a70
response_code = 200
uri = /network-bootstore-config
|
| 2453 | 2024-05-29T21:53:42.211Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:43075
|
| 2454 | 2024-05-29T21:53:42.212Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 46
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:43075
req_id = 40929abf-dab0-4de1-9a4b-88efc918fb98
response_code = 200
uri = /baseboard
|
| 2455 | 2024-05-29T21:53:55.027Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 3228
local_addr = [fd00:1122:3344:101::1]:61502
method = GET
remote_addr = [fd00:1122:3344:101::d]:45465
req_id = d22e6775-18e1-4407-9db2-941e7846dd7e
response_code = 200
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
uri = /67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2456 | 2024-05-29T21:54:00.197Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:39897
|
| 2457 | 2024-05-29T21:54:00.243Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 46563
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:39897
req_id = edcad443-8ace-4a58-a707-1a5a7200081e
response_code = 204
uri = /switch-ports
|
| 2458 | 2024-05-29T21:54:00.244Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 90
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:39897
req_id = 1c2046d7-16bc-4595-be1e-d2add6a6d48d
response_code = 200
uri = /network-bootstore-config
|
| 2459 | 2024-05-29T21:54:00.272Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:62983
|
| 2460 | 2024-05-29T21:54:00.317Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 44636
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:62983
req_id = e10de8d5-b479-4c8e-a935-88d92a7ce021
response_code = 204
uri = /switch-ports
|
| 2461 | 2024-05-29T21:54:00.317Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:56703
|
| 2462 | 2024-05-29T21:54:00.320Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 2704
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:62983
req_id = 729d26e6-e066-4b54-9b16-315fee8749d0
response_code = 200
uri = /network-bootstore-config
|
| 2463 | 2024-05-29T21:54:00.360Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 42541
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:56703
req_id = 588fcd68-0e63-4254-a410-67c2db4895a1
response_code = 204
uri = /switch-ports
|
| 2464 | 2024-05-29T21:54:00.360Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 164
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:56703
req_id = e94fe4ff-c514-49bc-9479-64e9789d8cc6
response_code = 200
uri = /network-bootstore-config
|
| 2465 | 2024-05-29T21:54:12.245Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45071
|
| 2466 | 2024-05-29T21:54:12.245Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 67
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45071
req_id = 17f33c60-4547-4854-957c-7e48c48dc532
response_code = 200
uri = /baseboard
|
| 2467 | 2024-05-29T21:54:25.073Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 601
local_addr = [fd00:1122:3344:101::1]:61502
method = GET
remote_addr = [fd00:1122:3344:101::d]:45465
req_id = e58c4f51-d93b-495f-a8cb-79fb43b5f48a
response_code = 200
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
uri = /67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2468 | 2024-05-29T21:54:30.517Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:52492
|
| 2469 | 2024-05-29T21:54:30.561Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 43819
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:52492
req_id = 2668e390-1911-4e21-a454-98118d2b8763
response_code = 204
uri = /switch-ports
|
| 2470 | 2024-05-29T21:54:30.561Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:64844
|
| 2471 | 2024-05-29T21:54:30.561Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:39351
|
| 2472 | 2024-05-29T21:54:30.562Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 230
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:52492
req_id = 3f85f0a3-9139-4e42-82f9-a1bb0bbdafb4
response_code = 200
uri = /network-bootstore-config
|
| 2473 | 2024-05-29T21:54:30.610Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 48249
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:64844
req_id = 18f87ec4-48ec-431e-b1b7-c62ba30dc0bd
response_code = 204
uri = /switch-ports
|
| 2474 | 2024-05-29T21:54:30.614Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 690
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:64844
req_id = b4f3f460-4f0c-49ef-903d-70c4558c562c
response_code = 200
uri = /network-bootstore-config
|
| 2475 | 2024-05-29T21:54:30.657Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 90578
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:39351
req_id = 44ddc1a5-a358-49ec-8726-c29ad8b3b49e
response_code = 204
uri = /switch-ports
|
| 2476 | 2024-05-29T21:54:30.657Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 127
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:39351
req_id = 45a4be73-4e96-4a16-a51b-27cbce0a1271
response_code = 200
uri = /network-bootstore-config
|
| 2477 | 2024-05-29T21:54:42.278Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:58866
|
| 2478 | 2024-05-29T21:54:42.278Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 92
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:58866
req_id = b07a2080-8771-45ca-b532-4b6014261a40
response_code = 200
uri = /baseboard
|
| 2479 | 2024-05-29T21:54:55.073Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 521
local_addr = [fd00:1122:3344:101::1]:61502
method = GET
remote_addr = [fd00:1122:3344:101::d]:45465
req_id = 8adbaf0f-9c96-4133-87d2-c12063981de4
response_code = 200
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
uri = /67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2480 | 2024-05-29T21:55:00.166Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:47345
|
| 2481 | 2024-05-29T21:55:00.213Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 46940
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:47345
req_id = 62ec964e-9e8e-4894-a18f-604f0dc99716
response_code = 204
uri = /switch-ports
|
| 2482 | 2024-05-29T21:55:00.214Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 94
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:47345
req_id = eb03c831-3831-4515-b4a7-3ad501072c54
response_code = 200
uri = /network-bootstore-config
|
| 2483 | 2024-05-29T21:55:00.273Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:50349
|
| 2484 | 2024-05-29T21:55:00.321Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 47596
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:50349
req_id = c9f9d59e-860d-4013-b826-f141a0c062c6
response_code = 204
uri = /switch-ports
|
| 2485 | 2024-05-29T21:55:00.321Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:59194
|
| 2486 | 2024-05-29T21:55:00.325Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 3539
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:50349
req_id = e062dbda-d97f-4742-af66-0ca672e44d92
response_code = 200
uri = /network-bootstore-config
|
| 2487 | 2024-05-29T21:55:00.367Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 46167
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:59194
req_id = c2321414-cb39-4ab0-adc2-7a68669a30cd
response_code = 204
uri = /switch-ports
|
| 2488 | 2024-05-29T21:55:00.368Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 133
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:59194
req_id = 1233bc39-8e41-4639-b7d1-333af6bb8276
response_code = 200
uri = /network-bootstore-config
|
| 2489 | 2024-05-29T21:55:12.311Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:41883
|
| 2490 | 2024-05-29T21:55:12.311Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 100
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:41883
req_id = aee9706d-7d1e-43ec-a92b-25b0eb0f1c24
response_code = 200
uri = /baseboard
|
| 2491 | 2024-05-29T21:55:25.076Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 2767
local_addr = [fd00:1122:3344:101::1]:61502
method = GET
remote_addr = [fd00:1122:3344:101::d]:45465
req_id = 625ef769-0423-4224-94c8-31b0d1bde9a6
response_code = 200
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
uri = /67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2492 | 2024-05-29T21:55:30.430Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:51417
|
| 2493 | 2024-05-29T21:55:30.430Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name("external-dns-ce802882-5f60-4fa0-8546-381dc464eec5"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name("external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name("nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name("nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name("nexus-73fd9e9c-3929-404c-966e-fc226ac774de"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2494 | 2024-05-29T21:55:30.430Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2495 | 2024-05-29T21:55:30.430Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2496 | 2024-05-29T21:55:30.430Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2497 | 2024-05-29T21:55:30.430Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2498 | 2024-05-29T21:55:30.430Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2499 | 2024-05-29T21:55:30.430Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2500 | 2024-05-29T21:55:30.430Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 393
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:51417
req_id = 05963163-14f1-4b3d-a628-aae5fc8b99c6
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2501 | 2024-05-29T21:55:30.447Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:58300
|
| 2502 | 2024-05-29T21:55:30.448Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name("external-dns-ce802882-5f60-4fa0-8546-381dc464eec5"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name("external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name("nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name("nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name("nexus-73fd9e9c-3929-404c-966e-fc226ac774de"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2503 | 2024-05-29T21:55:30.448Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2504 | 2024-05-29T21:55:30.448Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2505 | 2024-05-29T21:55:30.452Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2506 | 2024-05-29T21:55:30.452Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2507 | 2024-05-29T21:55:30.452Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2508 | 2024-05-29T21:55:30.452Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2509 | 2024-05-29T21:55:30.452Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 294
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:58300
req_id = 0bfe63dc-290e-4c1e-9eed-af9f0724572c
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2510 | 2024-05-29T21:55:30.454Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:50985
|
| 2511 | 2024-05-29T21:55:30.460Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:33484
|
| 2512 | 2024-05-29T21:55:30.460Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name("external-dns-ce802882-5f60-4fa0-8546-381dc464eec5"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name("external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name("nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name("nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name("nexus-73fd9e9c-3929-404c-966e-fc226ac774de"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2513 | 2024-05-29T21:55:30.460Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2514 | 2024-05-29T21:55:30.460Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2515 | 2024-05-29T21:55:30.460Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2516 | 2024-05-29T21:55:30.460Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2517 | 2024-05-29T21:55:30.460Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2518 | 2024-05-29T21:55:30.460Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2519 | 2024-05-29T21:55:30.460Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 331
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:33484
req_id = e34493db-4a3d-40fd-b626-0e622e401af8
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2520 | 2024-05-29T21:55:30.484Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:38339
|
| 2521 | 2024-05-29T21:55:30.499Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 44428
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:50985
req_id = 7aa649e4-f090-46c0-b19d-35e1f75033a1
response_code = 204
uri = /switch-ports
|
| 2522 | 2024-05-29T21:55:30.503Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 1238
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:50985
req_id = 6697d9cf-0ffa-4107-92aa-2e62aabd5784
response_code = 200
uri = /network-bootstore-config
|
| 2523 | 2024-05-29T21:55:30.503Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:50131
|
| 2524 | 2024-05-29T21:55:30.544Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 59061
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:38339
req_id = 5b6075ab-e1d2-492d-a730-20d4ce62c841
response_code = 204
uri = /switch-ports
|
| 2525 | 2024-05-29T21:55:30.548Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 649
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:38339
req_id = cf502cef-9a71-43f3-84eb-a184bb20072b
response_code = 200
uri = /network-bootstore-config
|
| 2526 | 2024-05-29T21:55:30.588Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 84816
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:50131
req_id = 3cb53acc-1230-488e-88a1-08d4d3536bc0
response_code = 204
uri = /switch-ports
|
| 2527 | 2024-05-29T21:55:30.588Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 182
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:50131
req_id = 884184c4-ce76-496f-9a18-09549c8500b4
response_code = 200
uri = /network-bootstore-config
|
| 2528 | 2024-05-29T21:55:42.343Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:37861
|
| 2529 | 2024-05-29T21:55:42.343Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 55
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:37861
req_id = 6ab940be-6cf2-4f33-ab62-a6b880fd666a
response_code = 200
uri = /baseboard
|
| 2530 | 2024-05-29T21:55:55.073Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 445
local_addr = [fd00:1122:3344:101::1]:61502
method = GET
remote_addr = [fd00:1122:3344:101::d]:45465
req_id = 4591703a-c955-4b99-a4b3-8b97eab1426f
response_code = 200
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
uri = /67239fc6-3779-4bd2-ac1a-314185fe431f
|
| 2531 | 2024-05-29T21:56:00.183Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:44174
|
| 2532 | 2024-05-29T21:56:00.230Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 47515
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:44174
req_id = 445233ce-2fae-43ca-ad08-861d8f092679
response_code = 204
uri = /switch-ports
|
| 2533 | 2024-05-29T21:56:00.230Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:62684
|
| 2534 | 2024-05-29T21:56:00.273Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 42924
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:62684
req_id = db74d891-dc50-4988-ba38-9da624cb7642
response_code = 204
uri = /switch-ports
|
| 2535 | 2024-05-29T21:56:00.273Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:49520
|
| 2536 | 2024-05-29T21:56:00.277Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 3495
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:44174
req_id = 438ed411-6e45-4045-ac6a-0e0c8b46c3fa
response_code = 200
uri = /network-bootstore-config
|
| 2537 | 2024-05-29T21:56:00.277Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 3760
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:62684
req_id = 9cf1b738-bf60-46c5-9642-6b9b1594f706
response_code = 200
uri = /network-bootstore-config
|
| 2538 | 2024-05-29T21:56:00.319Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 45788
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:49520
req_id = 7d46cbee-fcf5-4baf-94f5-b6746eb75e63
response_code = 204
uri = /switch-ports
|
| 2539 | 2024-05-29T21:56:00.319Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 111
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:49520
req_id = a88742e7-f1d3-4e94-93a7-d00354a2cc97
response_code = 200
uri = /network-bootstore-config
|
| 2540 | 2024-05-29T21:56:01.870Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:47563
|
| 2541 | 2024-05-29T21:56:01.882Z | INFO | SledAgent (InstanceManager): ensuring instance is registered
file = sled-agent/src/instance_manager.rs:546
hardware = InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname("localshark") }, nics: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(1417898))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: "nvme", name: "disk-e22f58cde08", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: bf2b630c-c4f1-4047-b6c6-bc780c1c4fc3, read_only_parent: Some(Volume { block_size: 512, id: f522685a-8c7c-4ea1-9a59-9d48f2b54b12, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 7f6e33e9-d32a-4fc1-a9f4-5c6deab98a7c, key: Some("fy067CHA0e6MAlslntAnx2FZZdEgCDSQLgfiRtdsXic="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::14]:19001", "[fd00:1122:3344:101::19]:19002", "[fd00:1122:3344:101::13]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: bf2b630c-c4f1-4047-b6c6-bc780c1c4fc3, key: Some("JBmRC8Qij6Z/zPLP3kshAVVOwqIOydO5xFm4vevNBeA="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::16]:19001", "[fd00:1122:3344:101::17]:19001", "[fd00:1122:3344:101::19]:19003"] } }] } }], cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAC3AK+9WL1YAAAAr71YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAtwCvvVi9WAAAAK+9WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5OGZhYzRlZC04NjFmLTQ2NWQtYjczOS03ODVjNjVlMjFiOGEiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQ2lJeDZ5SksrRmUyUmhDMkZheUptWGRMOFdoby9qb3NLQ2JtMms2Z0JWdCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") }
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
instance_runtime = InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }
metadata = InstanceMetadata { silo_id: 7f285fd5-6333-41bb-876e-2e81f40964f1, project_id: 0289c9e5-a51b-4d7a-ad72-2f0ec456b725 }
propolis_addr = [fd00:1122:3344:101::1:0]:12400
propolis_id = 8ad579b2-dda9-4e63-8490-567c3ccd27af
vmm_runtime = VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-29T21:56:01.397941Z }
|
| 2542 | 2024-05-29T21:56:01.882Z | INFO | SledAgent (InstanceManager): registering new instance
file = sled-agent/src/instance_manager.rs:580
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2543 | 2024-05-29T21:56:01.882Z | INFO | SledAgent (InstanceManager): initializing new Instance
file = sled-agent/src/instance.rs:938
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
propolis_id = 8ad579b2-dda9-4e63-8490-567c3ccd27af
state = InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname("localshark") }, nics: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(1417898))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: "nvme", name: "disk-e22f58cde08", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: bf2b630c-c4f1-4047-b6c6-bc780c1c4fc3, read_only_parent: Some(Volume { block_size: 512, id: f522685a-8c7c-4ea1-9a59-9d48f2b54b12, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 7f6e33e9-d32a-4fc1-a9f4-5c6deab98a7c, key: Some("fy067CHA0e6MAlslntAnx2FZZdEgCDSQLgfiRtdsXic="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::14]:19001", "[fd00:1122:3344:101::19]:19002", "[fd00:1122:3344:101::13]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: bf2b630c-c4f1-4047-b6c6-bc780c1c4fc3, key: Some("JBmRC8Qij6Z/zPLP3kshAVVOwqIOydO5xFm4vevNBeA="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::16]:19001", "[fd00:1122:3344:101::17]:19001", "[fd00:1122:3344:101::19]:19003"] } }] } }], cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAC3AK+9WL1YAAAAr71YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAtwCvvVi9WAAAAK+9WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5OGZhYzRlZC04NjFmLTQ2NWQtYjczOS03ODVjNjVlMjFiOGEiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQ2lJeDZ5SksrRmUyUmhDMkZheUptWGRMOFdoby9qb3NLQ2JtMms2Z0JWdCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-29T21:56:01.397941Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }
|
| 2544 | 2024-05-29T21:56:01.883Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 12835
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:47563
req_id = 8a307cfd-37c4-4fde-94cb-ee8c07c8780d
response_code = 200
uri = /instances/98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2545 | 2024-05-29T21:56:01.975Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:52578
|
| 2546 | 2024-05-29T21:56:01.976Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 622
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:52578
req_id = d676da51-be25-491d-8609-3d866ffad6d6
response_code = 200
uri = /instances/98fac4ed-861f-465d-b739-785c65e21b8a/state
|
| 2547 | 2024-05-29T21:56:02.000Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte6", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 250, 74, 86]), slot: 0, vni: Vni { inner: 1417898 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: "vopte6" } }
|
| 2548 | 2024-05-29T21:56:02.000Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2549 | 2024-05-29T21:56:02.032Z | INFO | SledAgent (InstanceManager): Configuring new Omicron zone: oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af
file = illumos-utils/src/zone.rs:315
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2550 | 2024-05-29T21:56:02.054Z | INFO | SledAgent (InstanceManager): Installing Omicron zone: oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af
file = illumos-utils/src/zone.rs:354
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2551 | 2024-05-29T21:56:03.175Z | INFO | SledAgent (InstanceManager): Profile for oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="system/illumos/propolis-server">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlInstance0"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::1:0"/>
<propval type="astring" name="listen_port" value="12400"/>
<propval type="astring" name="metric_addr" value="[fd00:1122:3344:101::b]:12221"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2552 | 2024-05-29T21:56:03.175Z | INFO | SledAgent (InstanceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
zone = oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af
|
| 2553 | 2024-05-29T21:56:03.358Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af") failed: Property not found. retry in 74.159154ms
file = illumos-utils/src/svc.rs:36
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
zone = oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af
|
| 2554 | 2024-05-29T21:56:03.446Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af") failed: Property not found. retry in 121.151009ms
file = illumos-utils/src/svc.rs:36
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
zone = oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af
|
| 2555 | 2024-05-29T21:56:03.587Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af") failed: Property not found. retry in 197.661154ms
file = illumos-utils/src/svc.rs:36
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
zone = oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af
|
| 2556 | 2024-05-29T21:56:03.799Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af") failed: Property not found. retry in 498.341732ms
file = illumos-utils/src/svc.rs:36
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
zone = oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af
|
| 2557 | 2024-05-29T21:56:04.306Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af") failed: Property not found. retry in 844.935012ms
file = illumos-utils/src/svc.rs:36
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
zone = oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af
|
| 2558 | 2024-05-29T21:56:05.162Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af") failed: Property not found. retry in 1.239328232s
file = illumos-utils/src/svc.rs:36
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
zone = oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af
|
| 2559 | 2024-05-29T21:56:06.412Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af") failed: Property not found. retry in 854.836921ms
file = illumos-utils/src/svc.rs:36
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
zone = oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af
|
| 2560 | 2024-05-29T21:56:07.277Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af") failed: Property not found. retry in 1.421988085s
file = illumos-utils/src/svc.rs:36
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
zone = oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af
|
| 2561 | 2024-05-29T21:56:08.727Z | INFO | SledAgent (InstanceManager): Started propolis in zone: oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af
file = sled-agent/src/instance.rs:1423
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2562 | 2024-05-29T21:56:08.735Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af") failed: Property not found. retry in 52.457579ms
file = illumos-utils/src/svc.rs:36
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2563 | 2024-05-29T21:56:08.796Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af") failed: Property not found. retry in 75.07528ms
file = illumos-utils/src/svc.rs:36
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2564 | 2024-05-29T21:56:08.881Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af") failed: Property not found. retry in 267.932745ms
file = illumos-utils/src/svc.rs:36
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2565 | 2024-05-29T21:56:09.159Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af") failed: Property not found. retry in 546.363566ms
file = illumos-utils/src/svc.rs:36
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2566 | 2024-05-29T21:56:09.715Z | INFO | SledAgent (InstanceManager): Propolis SMF service is online
file = sled-agent/src/instance.rs:1432
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2567 | 2024-05-29T21:56:09.749Z | INFO | SledAgent (InstanceManager): Propolis HTTP server online
file = sled-agent/src/instance.rs:1446
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2568 | 2024-05-29T21:56:09.750Z | INFO | SledAgent (InstanceManager): Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAC3AK+9WL1YAAAAr71YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAtwCvvVi9WAAAAK+9WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5OGZhYzRlZC04NjFmLTQ2NWQtYjczOS03ODVjNjVlMjFiOGEiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQ2lJeDZ5SksrRmUyUmhDMkZheUptWGRMOFdoby9qb3NLQ2JtMms2Z0JWdCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"), disks: [DiskRequest { device: "nvme", name: "disk-e22f58cde08", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: bf2b630c-c4f1-4047-b6c6-bc780c1c4fc3, read_only_parent: Some(Volume { block_size: 512, id: f522685a-8c7c-4ea1-9a59-9d48f2b54b12, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 7f6e33e9-d32a-4fc1-a9f4-5c6deab98a7c, key: Some("fy067CHA0e6MAlslntAnx2FZZdEgCDSQLgfiRtdsXic="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::14]:19001", "[fd00:1122:3344:101::19]:19002", "[fd00:1122:3344:101::13]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: bf2b630c-c4f1-4047-b6c6-bc780c1c4fc3, key: Some("JBmRC8Qij6Z/zPLP3kshAVVOwqIOydO5xFm4vevNBeA="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::16]:19001", "[fd00:1122:3344:101::17]:19001", "[fd00:1122:3344:101::19]:19003"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: "vopte6", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: "Test description", id: 98fac4ed-861f-465d-b739-785c65e21b8a, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 0289c9e5-a51b-4d7a-ad72-2f0ec456b725, silo_id: 7f285fd5-6333-41bb-876e-2e81f40964f1 }, name: "localshark", vcpus: 2 } }
file = sled-agent/src/instance.rs:685
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2569 | 2024-05-29T21:56:09.859Z | INFO | SledAgent (InstanceManager): result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })
file = sled-agent/src/instance.rs:687
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2570 | 2024-05-29T21:56:09.859Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-05-29T21:56:09.859908568Z }
file = sled-agent/src/instance.rs:569
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2571 | 2024-05-29T21:56:09.859Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:591
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
new_instance_state = InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }
new_vmm_state = VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-29T21:56:09.859908568Z }
propolis_id = 8ad579b2-dda9-4e63-8490-567c3ccd27af
|
| 2572 | 2024-05-29T21:56:09.859Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }, propolis_id: 8ad579b2-dda9-4e63-8490-567c3ccd27af, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-29T21:56:09.859908568Z } }
|
| 2573 | 2024-05-29T21:56:09.917Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-05-29T21:56:09.917923640Z }
file = sled-agent/src/instance.rs:569
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2574 | 2024-05-29T21:56:09.917Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:591
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
new_instance_state = InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }
new_vmm_state = VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-29T21:56:09.917923640Z }
propolis_id = 8ad579b2-dda9-4e63-8490-567c3ccd27af
|
| 2575 | 2024-05-29T21:56:09.917Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }, propolis_id: 8ad579b2-dda9-4e63-8490-567c3ccd27af, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-29T21:56:09.917923640Z } }
|
| 2576 | 2024-05-29T21:56:12.377Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:38695
|
| 2577 | 2024-05-29T21:56:12.377Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 54
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:38695
req_id = d3270bba-7662-46b1-a4b1-c40ed186317b
response_code = 200
uri = /baseboard
|
| 2578 | 2024-05-29T21:56:18.749Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:46421
|
| 2579 | 2024-05-29T21:56:18.749Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 592
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:46421
req_id = 2c4672fe-1490-4971-b3e0-f05ee7c2acd7
response_code = 200
uri = /instances/98fac4ed-861f-465d-b739-785c65e21b8a/state
|
| 2580 | 2024-05-29T21:56:18.750Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-05-29T21:56:18.750043330Z }
file = sled-agent/src/instance.rs:569
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2581 | 2024-05-29T21:56:18.750Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:591
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
new_instance_state = InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }
new_vmm_state = VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-29T21:56:18.750043330Z }
propolis_id = 8ad579b2-dda9-4e63-8490-567c3ccd27af
|
| 2582 | 2024-05-29T21:56:18.750Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }, propolis_id: 8ad579b2-dda9-4e63-8490-567c3ccd27af, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-29T21:56:18.750043330Z } }
|
| 2583 | 2024-05-29T21:56:18.776Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-05-29T21:56:18.776758522Z }
file = sled-agent/src/instance.rs:569
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2584 | 2024-05-29T21:56:18.776Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:591
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
new_instance_state = InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-29T21:56:18.776758522Z }
new_vmm_state = VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-05-29T21:56:18.776758522Z }
propolis_id = 8ad579b2-dda9-4e63-8490-567c3ccd27af
|
| 2585 | 2024-05-29T21:56:18.776Z | INFO | SledAgent (InstanceManager): terminating VMM that has exited
file = sled-agent/src/instance.rs:606
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2586 | 2024-05-29T21:56:18.776Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2587 | 2024-05-29T21:56:18.778Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
| 2588 | 2024-05-29T21:56:18.778Z | INFO | SledAgent (ZoneBundler): creating zone bundle
context = ZoneBundleContext { storage_dirs: ["/pool/int/800d63a5-84ab-4e81-94a7-03a851147655/debug/bundle/zone", "/pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/debug/bundle/zone"], cause: TerminatedInstance, extra_log_dirs: ["/pool/ext/aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c/crypt/debug", "/pool/ext/8f301520-a970-47c3-90c3-bd3006e5615c/crypt/debug", "/pool/ext/cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crypt/debug", "/pool/ext/b5814935-58c2-4e3c-9137-4cbaa1ee9fab/crypt/debug", "/pool/ext/020b43a6-e481-4304-8596-25774e86f14b/crypt/debug", "/pool/ext/4532d18b-4909-4537-9ce6-98b74fb663f5/crypt/debug", "/pool/ext/8d81843f-0d2d-4c69-8d17-45004d7cf484/crypt/debug", "/pool/ext/59fcaf06-997c-43d9-9256-26a5eda3000f/crypt/debug", "/pool/ext/6b416693-d467-424a-86a1-703b05ad3101/crypt/debug"] }
file = sled-agent/src/zone_bundle.rs:437
zone_name = oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af
|
| 2589 | 2024-05-29T21:56:19.503Z | INFO | SledAgent (ZoneBundler): finished zone bundle
file = sled-agent/src/zone_bundle.rs:1230
metadata = ZoneBundleMetadata { id: ZoneBundleId { zone_name: "oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af", bundle_id: dbd3bc50-8793-4e45-a865-c157e7a3a9d6 }, time_created: 2024-05-29T21:56:18.779043850Z, version: 0, cause: TerminatedInstance }
|
| 2590 | 2024-05-29T21:56:19.503Z | WARN | SledAgent (InstanceManager): Halting and removing zone: oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af
file = sled-agent/src/instance.rs:780
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2591 | 2024-05-29T21:56:21.257Z | INFO | SledAgent (InstanceManager): halt_and_remove_logged: Previous zone state: Running
file = illumos-utils/src/zone.rs:269
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2592 | 2024-05-29T21:56:21.282Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-29T21:56:18.776758522Z }, propolis_id: 8ad579b2-dda9-4e63-8490-567c3ccd27af, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-29T21:56:21.282836850Z } }
|
| 2593 | 2024-05-29T21:56:21.355Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:62474
|
| 2594 | 2024-05-29T21:56:21.362Z | WARN | SledAgent (PortManager): unset_virtual_nic_host unimplmented
file = illumos-utils/src/opte/port_manager.rs:620
|
| 2595 | 2024-05-29T21:56:21.362Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 6554
local_addr = [fd00:1122:3344:101::1]:12345
method = DELETE
remote_addr = [fd00:1122:3344:101::a]:62474
req_id = ecab2288-24d7-402d-97b0-2d257482c080
response_code = 204
uri = /v2p/ab35f591-930b-49b8-b3e0-a510c19d1b2c
|
| 2596 | 2024-05-29T21:56:21.661Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-29T21:56:18.776758522Z }, propolis_id: 8ad579b2-dda9-4e63-8490-567c3ccd27af, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-29T21:56:21.282836850Z } }
|
| 2597 | 2024-05-29T21:56:21.674Z | INFO | SledAgent (InstanceManager): State monitoring task complete
file = sled-agent/src/instance.rs:723
instance_id = 98fac4ed-861f-465d-b739-785c65e21b8a
|
| 2598 | 2024-05-29T21:56:25.074Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878
latency_us = 608
local_addr = [fd00:1122:3344:101::1]:61502
method = GET
remote_addr = [fd00:1122:3344:101::d]:45465
req_id = f360186f-f67a-4780-8612-86aaff77f27c
response_code = 200
sled_id = 67239fc6-3779-4bd2-ac1a-314185fe431f
uri = /67239fc6-3779-4bd2-ac1a-314185fe431f
|