1 | | [ May 15 07:08:05 Enabled. ] |
2 | | [ May 15 07:08:05 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] |
3 | | [ May 15 07:08:05 Method "start" exited with status 0. ] |
4 | | [ May 15 07:08:05 Rereading configuration. ] |
5 | | [ May 15 07:08:05 No 'refresh' method defined. Treating as :true. ] |
6 | | note: configured to log to "/dev/stdout" |
7 | 2024-05-15T07:08:05.477Z | INFO | SledAgent: Ensuring contract reaper thread
file = illumos-utils/src/running_zone.rs:163
|
8 | 2024-05-15T07:08:05.477Z | INFO | SledAgent (BootstrapAgentStartup): finding links ["net0", "net1"]
file = sled-agent/src/bootstrap/pre_server.rs:255
|
9 | 2024-05-15T07:08:05.530Z | INFO | SledAgent (BootstrapAgentStartup): Starting mg-ddm service
addr-objs = [AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]
file = sled-agent/src/bootstrap/pre_server.rs:262
|
10 | 2024-05-15T07:08:05.530Z | INFO | SledAgent (BootstrapAgentStartup): Importing mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:42
path = /opt/oxide/mg-ddm/pkg/ddm/manifest.xml
|
11 | 2024-05-15T07:08:05.641Z | INFO | SledAgent (BootstrapAgentStartup): Setting mg-ddm interfaces
file = sled-agent/src/bootstrap/maghemite.rs:50
interfaces = ("net0/ll" "net1/ll")
|
12 | 2024-05-15T07:08:05.650Z | INFO | SledAgent (BootstrapAgentStartup): Enabling mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:56
|
13 | 2024-05-15T07:08:05.658Z | INFO | SledAgent (BootstrapAgentStartup): Ensuring zfs key directory exists
file = sled-agent/src/bootstrap/pre_server.rs:272
path = /var/run/oxide/
|
14 | 2024-05-15T07:08:06.258Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }
|
15 | 2024-05-15T07:08:06.397Z | INFO | SledAgent: Starting KeyManager
file = sled-agent/src/long_running_tasks.rs:119
|
16 | 2024-05-15T07:08:06.397Z | INFO | SledAgent: Starting StorageManager
file = sled-agent/src/long_running_tasks.rs:131
|
17 | 2024-05-15T07:08:06.397Z | INFO | SledAgent: Starting StorageMonitor
file = sled-agent/src/long_running_tasks.rs:141
|
18 | 2024-05-15T07:08:06.397Z | INFO | SledAgent: Starting HardwareManager
file = sled-agent/src/long_running_tasks.rs:162
nongimlet_observed_disks = []
sled_mode = Scrimlet { asic: SoftNpuZone }
|
19 | 2024-05-15T07:08:06.397Z | INFO | SledAgent (DumpSetup-worker): DumpSetup poll loop started.
file = sled-agent/src/dump_setup.rs:506
|
20 | 2024-05-15T07:08:06.397Z | INFO | SledAgent (HardwareManager): Creating HardwareManager
file = sled-hardware/src/illumos/mod.rs:625
|
21 | 2024-05-15T07:08:06.400Z | INFO | SledAgent (HardwareManager): Generated i86pc baseboard Pc { identifier: "sock", model: "i86pc" }
file = sled-hardware/src/illumos/mod.rs:535
|
22 | 2024-05-15T07:08:06.400Z | WARN | SledAgent (HardwareManager): Device is not a Gimlet (i86pc), proceeding with null hardware view
file = sled-hardware/src/illumos/mod.rs:674
|
23 | 2024-05-15T07:08:06.400Z | INFO | SledAgent: Starting HardwareMonitor
file = sled-agent/src/long_running_tasks.rs:176
|
24 | 2024-05-15T07:08:06.400Z | INFO | SledAgent (HardwareManager): Monitoring for hardware updates
file = sled-hardware/src/illumos/mod.rs:733
|
25 | 2024-05-15T07:08:06.400Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/m2_0.vdev
|
26 | 2024-05-15T07:08:06.400Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
disks = {}
file = sled-agent/src/hardware_monitor.rs:262
underlay_network_info = None
|
27 | 2024-05-15T07:08:06.401Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
28 | 2024-05-15T07:08:06.401Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }
|
29 | 2024-05-15T07:08:06.401Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:112
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }
path = /scratch/m2_0.vdev
|
30 | 2024-05-15T07:08:06.413Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev
file = sled-hardware/src/disk.rs:304
|
31 | 2024-05-15T07:08:06.413Z | INFO | SledAgent (StorageResources): Formatting zpool with generated ID
file = sled-hardware/src/disk.rs:315
id = c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool)
|
32 | 2024-05-15T07:08:06.514Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal }
|
33 | 2024-05-15T07:08:06.714Z | INFO | SledAgent (StorageManager): Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }
file = sled-storage/src/manager.rs:381
|
34 | 2024-05-15T07:08:06.714Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/m2_1.vdev
|
35 | 2024-05-15T07:08:06.714Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }
|
36 | 2024-05-15T07:08:06.714Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }
|
37 | 2024-05-15T07:08:06.714Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
38 | 2024-05-15T07:08:06.714Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }
|
39 | 2024-05-15T07:08:06.714Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:112
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }
path = /scratch/m2_1.vdev
|
40 | 2024-05-15T07:08:06.722Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev
file = sled-hardware/src/disk.rs:304
|
41 | 2024-05-15T07:08:06.722Z | INFO | SledAgent (StorageResources): Formatting zpool with generated ID
file = sled-hardware/src/disk.rs:315
id = 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool)
|
42 | 2024-05-15T07:08:06.806Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal }
|
43 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
44 | 2024-05-15T07:08:07.003Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_0.vdev
|
45 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
46 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
47 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
48 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
|
49 | 2024-05-15T07:08:07.003Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_1.vdev
|
50 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
51 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
52 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
53 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
54 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
|
55 | 2024-05-15T07:08:07.003Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_2.vdev
|
56 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
57 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
58 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
59 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
60 | 2024-05-15T07:08:07.003Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
|
61 | 2024-05-15T07:08:07.004Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_3.vdev
|
62 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
63 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
64 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
|
65 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
66 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
67 | 2024-05-15T07:08:07.004Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_4.vdev
|
68 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
69 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, variant: U2, slot: 1030 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
70 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
71 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
|
72 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
73 | 2024-05-15T07:08:07.004Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_5.vdev
|
74 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, variant: U2, slot: 1031 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
75 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
76 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
|
77 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
78 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
79 | 2024-05-15T07:08:07.004Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_6.vdev
|
80 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
81 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }, variant: U2, slot: 1032 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
82 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
83 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
|
84 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
85 | 2024-05-15T07:08:07.004Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_7.vdev
|
86 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
87 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }, variant: U2, slot: 1033 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
88 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
89 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
90 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
|
91 | 2024-05-15T07:08:07.004Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:232
vdev = /scratch/u2_8.vdev
|
92 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }, variant: U2, slot: 1034 }), tx: ... }
file = sled-storage/src/manager.rs:381
|
93 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:494
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
|
94 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
95 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
96 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
97 | 2024-05-15T07:08:07.004Z | INFO | SledAgent: Waiting for boot disk
file = sled-agent/src/long_running_tasks.rs:91
|
98 | 2024-05-15T07:08:07.004Z | INFO | SledAgent: Found boot disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }
file = sled-agent/src/long_running_tasks.rs:93
|
99 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
100 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
101 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
102 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
103 | 2024-05-15T07:08:07.004Z | INFO | SledAgent: Starting Bootstore
file = sled-agent/src/long_running_tasks.rs:200
|
104 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (bootstore): No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
105 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (bootstore): No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
106 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (bootstore): No ledger found. Loading Fsm::State as Uninitialized
file = bootstore/src/schemes/v0/storage.rs:83
peer_id = pc-sock-i86pc
|
107 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (bootstore): No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/cluster/bootstore-network-config.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
108 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (bootstore): No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/cluster/bootstore-network-config.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
109 | 2024-05-15T07:08:07.004Z | INFO | SledAgent (bootstore): No ledger found for network config
file = bootstore/src/schemes/v0/storage.rs:146
peer_id = pc-sock-i86pc
|
110 | 2024-05-15T07:08:07.004Z | INFO | SledAgent: Starting Bootstore DDMD poller
file = sled-agent/src/long_running_tasks.rs:205
|
111 | 2024-05-15T07:08:07.004Z | INFO | SledAgent: Starting ZoneBundler related tasks
file = sled-agent/src/long_running_tasks.rs:220
|
112 | 2024-05-15T07:08:07.015Z | INFO | SledAgent (ServiceManager): Creating ServiceManager
file = sled-agent/src/services.rs:711
|
113 | 2024-05-15T07:08:07.015Z | INFO | SledAgent (auto-cleanup-task): top of bundle cleanup loop
file = sled-agent/src/zone_bundle.rs:289
next_cleanup = Instant { tv_sec: 1251, tv_nsec: 643535330 }
time_to_next_cleanup = 599.999886978s
|
114 | 2024-05-15T07:08:07.015Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3617
|
115 | 2024-05-15T07:08:07.015Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
116 | 2024-05-15T07:08:07.015Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/sled-agent-request.json
file = common/src/ledger.rs:188
|
117 | 2024-05-15T07:08:07.015Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/sled-agent-request.json
file = common/src/ledger.rs:188
|
118 | 2024-05-15T07:08:07.018Z | INFO | SledAgent (dropshot (BootstrapAgent)): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
|
119 | 2024-05-15T07:08:07.018Z | INFO | SledAgent (SledAgentSprocketsServer): Started listening
file = sled-agent/src/bootstrap/sprockets_server.rs:44
local_addr = [fdb0:18c0:4d0d:a02a::1]:12346
|
120 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (RSS): Injecting RSS configuration: RackInitializeRequest {
trust_quorum_peers: None,
bootstrap_discovery: OnlyOurs,
ntp_servers: [
"0.pool.ntp.org",
],
dns_servers: [
1.1.1.1,
9.9.9.9,
],
internal_services_ip_pool_ranges: [
V4(
Ipv4Range {
first: 10.151.2.164,
last: 10.151.2.173,
},
),
],
external_dns_ips: [
10.151.2.164,
10.151.2.165,
],
external_dns_zone_name: "oxide.test",
external_certificates: "<redacted>",
recovery_silo: RecoverySiloConfig {
silo_name: Name(
"recovery",
),
user_name: UserId(
"recovery",
),
user_password_hash: NewPasswordHash(
PasswordHashString {
string: "$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY",
encoding: B64,
},
),
},
rack_network_config: RackNetworkConfigV1 {
rack_subnet: Ipv6Network {
addr: fd00:1122:3344:100::,
prefix: 56,
},
infra_ip_first: 10.151.2.174,
infra_ip_last: 10.151.2.174,
ports: [
PortConfigV1 {
routes: [
RouteConfig {
destination: V4(
Ipv4Network {
addr: 0.0.0.0,
prefix: 0,
},
),
nexthop: 10.151.2.100,
vlan_id: None,
},
],
addresses: [
V4(
Ipv4Network {
addr: 10.151.2.174,
prefix: 24,
},
),
],
switch: Switch0,
port: "qsfp0",
uplink_port_speed: Speed40G,
uplink_port_fec: None,
bgp_peers: [],
autoneg: false,
},
],
bgp: [],
bfd: [],
},
allowed_source_ips: Any,
}
file = sled-agent/src/rack_setup/service.rs:1003
|
121 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
122 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
123 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (RSS): No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/rss-plan-completed.marker
file = common/src/ledger.rs:188
|
124 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (RSS): No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/rss-plan-completed.marker
file = common/src/ledger.rs:188
|
125 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (RSS): RSS configuration has not been fully applied yet
file = sled-agent/src/rack_setup/service.rs:1063
|
126 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
127 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (RSS): No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/rss-sled-plan.json
file = common/src/ledger.rs:188
|
128 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (RSS): No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/rss-sled-plan.json
file = common/src/ledger.rs:188
|
129 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (RSS): Creating new allocation plan
file = sled-agent/src/rack_setup/service.rs:1104
|
130 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (RSS): Serializing plan
file = sled-agent/src/rack_setup/plan/sled.rs:118
|
131 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (RSS): Creating plan for the sled at fdb0:18c0:4d0d:a02a::1
file = sled-agent/src/rack_setup/plan/sled.rs:91
|
132 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
133 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.rss-sled-plan.json.tmp
file = common/src/ledger.rs:199
|
134 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.rss-sled-plan.json.tmp
file = common/src/ledger.rs:199
|
135 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (RSS): Sled plan written to storage
file = sled-agent/src/rack_setup/plan/sled.rs:138
|
136 | 2024-05-15T07:08:07.019Z | INFO | SledAgent (RSS): Writing Rack Network Configuration to bootstore
file = sled-agent/src/rack_setup/service.rs:1137
|
137 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 1, current_generation: 0
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
138 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
139 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
140 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 1
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-sock-i86pc
|
141 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (BootstrapAgentRssHandler): Received initialization request from RSS
file = sled-agent/src/bootstrap/rss_handle.rs:217
request = StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: acb2a796-8e9f-4811-b601-726a12d79b0b, rack_id: 89a4c48a-3dff-478b-98cb-9f441a913804, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
142 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (SledAgentSprocketsServer): Accepted connection
file = sled-agent/src/bootstrap/sprockets_server.rs:66
remote_addr = [fdb0:18c0:4d0d:a02a::1]:53471
|
143 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (SledAgentMain): Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: acb2a796-8e9f-4811-b601-726a12d79b0b, rack_id: 89a4c48a-3dff-478b-98cb-9f441a913804, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }
file = sled-agent/src/bootstrap/server.rs:360
|
144 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (SledAgentMain): KeyManager: using hardcoded secret retriever
file = sled-agent/src/bootstrap/server.rs:375
|
145 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (StorageManager): Received KeyManagerReady
file = sled-storage/src/manager.rs:381
|
146 | 2024-05-15T07:08:07.020Z | INFO | SledAgent: setting up sled agent server
file = sled-agent/src/server.rs:44
|
147 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (StorageManager): Loading storage config
file = sled-storage/src/manager.rs:530
|
148 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
149 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = load_ledger
|
150 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }
|
151 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = load_ledger
|
152 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (StorageManager): No ledger of physical disks exists
file = sled-storage/src/manager.rs:519
|
153 | 2024-05-15T07:08:07.020Z | INFO | SledAgent (StorageManager): KeyManager ready, but no ledger detected
file = sled-storage/src/manager.rs:547
|
154 | 2024-05-15T07:08:07.023Z | INFO | SledAgent (ServiceManager): Enabling switch zone (new)
file = sled-agent/src/services.rs:3849
|
155 | 2024-05-15T07:08:07.030Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
|
156 | 2024-05-15T07:08:07.041Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
|
157 | 2024-05-15T07:08:07.051Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
|
158 | 2024-05-15T07:08:07.060Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
|
159 | 2024-05-15T07:08:07.069Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
|
160 | 2024-05-15T07:08:07.077Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
|
161 | 2024-05-15T07:08:07.086Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
|
162 | 2024-05-15T07:08:07.094Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
|
163 | 2024-05-15T07:08:07.102Z | INFO | SledAgent (StorageManager): Zpool does not exist
err = ZpoolDoesNotExist
file = sled-storage/src/manager.rs:603
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
|
164 | 2024-05-15T07:08:07.102Z | INFO | SledAgent (StorageManager): No disks to be automatically managed
file = sled-storage/src/manager.rs:558
|
165 | 2024-05-15T07:08:07.102Z | INFO | SledAgent (StorageManager): automatically managing disks
file = sled-storage/src/manager.rs:370
|
166 | 2024-05-15T07:08:07.102Z | INFO | SledAgent (StorageResources): Synchronizing disk managment
file = sled-storage/src/resources.rs:339
|
167 | 2024-05-15T07:08:07.102Z | INFO | SledAgent (StorageManager): Successfully synchronized disks without error
file = sled-storage/src/manager.rs:449
result = DisksManagementResult { status: [] }
|
168 | 2024-05-15T07:08:07.102Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
169 | 2024-05-15T07:08:07.114Z | INFO | SledAgent: SledAgent::new(..) starting
file = sled-agent/src/sled_agent.rs:355
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
170 | 2024-05-15T07:08:07.114Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
171 | 2024-05-15T07:08:07.114Z | INFO | SledAgent: Requested swap device of size 64 GiB
file = sled-agent/src/sled_agent.rs:367
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
172 | 2024-05-15T07:08:07.114Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
173 | 2024-05-15T07:08:07.120Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 79.423477ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "701c516a-e11d-4925-a578-f7ae5782a9d5", "content-length": "124", "date": "Wed, 15 May 2024 07:08:06 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "701c516a-e11d-4925-a578-f7ae5782a9d5" }
file = clients/ddm-admin-client/src/lib.rs:178
|
174 | 2024-05-15T07:08:07.124Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:315
|
175 | 2024-05-15T07:08:07.127Z | INFO | SledAgent: attempting to create encrypted zvol: name="oxi_c3ed08c3-e0e1-43f2-b040-3c4755a2167c/swap", size_gb=64
file = sled-agent/src/swap_device.rs:153
|
176 | 2024-05-15T07:08:07.142Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:354
|
177 | 2024-05-15T07:08:07.150Z | INFO | SledAgent: successfully created encrypted zvol: name="oxi_c3ed08c3-e0e1-43f2-b040-3c4755a2167c/swap", size_gb=64
file = sled-agent/src/swap_device.rs:249
|
178 | 2024-05-15T07:08:07.150Z | INFO | SledAgent: adding swap device: swapname="/dev/zvol/dsk/oxi_c3ed08c3-e0e1-43f2-b040-3c4755a2167c/swap"
file = sled-agent/src/swap_device.rs:96
|
179 | 2024-05-15T07:08:07.155Z | INFO | SledAgent: Mounting backing filesystems
file = sled-agent/src/sled_agent.rs:382
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
180 | 2024-05-15T07:08:07.155Z | INFO | SledAgent (BackingFs): Processing fmd
file = sled-agent/src/backing_fs.rs:126
|
181 | 2024-05-15T07:08:07.155Z | INFO | SledAgent (BackingFs): Ensuring dataset oxi_c3ed08c3-e0e1-43f2-b040-3c4755a2167c/backing/fmd
file = sled-agent/src/backing_fs.rs:136
|
182 | 2024-05-15T07:08:07.200Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
183 | 2024-05-15T07:08:07.201Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 121.646521ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "85a9edee-feb2-4397-89eb-8b3d4180f52a", "content-length": "124", "date": "Wed, 15 May 2024 07:08:06 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "85a9edee-feb2-4397-89eb-8b3d4180f52a" }
file = clients/ddm-admin-client/src/lib.rs:178
|
184 | 2024-05-15T07:08:07.229Z | INFO | SledAgent (BackingFs): Stopping service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:170
|
185 | 2024-05-15T07:08:07.324Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
186 | 2024-05-15T07:08:07.325Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 171.894585ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "36ca5c40-66b3-458a-b380-61b68a92cb9d", "content-length": "124", "date": "Wed, 15 May 2024 07:08:07 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "36ca5c40-66b3-458a-b380-61b68a92cb9d" }
file = clients/ddm-admin-client/src/lib.rs:178
|
187 | 2024-05-15T07:08:07.448Z | INFO | SledAgent (BackingFs): Mounting oxi_c3ed08c3-e0e1-43f2-b040-3c4755a2167c/backing/fmd on /var/fm/fmd
file = sled-agent/src/backing_fs.rs:178
|
188 | 2024-05-15T07:08:07.461Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/rsrc
file = sled-agent/src/backing_fs.rs:186
|
189 | 2024-05-15T07:08:07.461Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/ckpt
file = sled-agent/src/backing_fs.rs:186
|
190 | 2024-05-15T07:08:07.461Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/xprt
file = sled-agent/src/backing_fs.rs:186
|
191 | 2024-05-15T07:08:07.461Z | INFO | SledAgent (BackingFs): Starting service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:192
|
192 | 2024-05-15T07:08:07.497Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
193 | 2024-05-15T07:08:07.498Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 95.619005ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a004534c-1236-4bc3-bd6a-633835eb5c1f", "content-length": "124", "date": "Wed, 15 May 2024 07:08:07 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a004534c-1236-4bc3-bd6a-633835eb5c1f" }
file = clients/ddm-admin-client/src/lib.rs:178
|
194 | 2024-05-15T07:08:07.595Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
195 | 2024-05-15T07:08:07.595Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 194.830069ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "bd9f34c8-02be-4b68-884d-0c320f377d64", "content-length": "124", "date": "Wed, 15 May 2024 07:08:07 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "bd9f34c8-02be-4b68-884d-0c320f377d64" }
file = clients/ddm-admin-client/src/lib.rs:178
|
196 | 2024-05-15T07:08:07.792Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
197 | 2024-05-15T07:08:07.793Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 333.055205ms
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e67acb4d-63d5-4730-9a2a-fcffedf5e699", "content-length": "124", "date": "Wed, 15 May 2024 07:08:07 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e67acb4d-63d5-4730-9a2a-fcffedf5e699" }
file = clients/ddm-admin-client/src/lib.rs:178
|
198 | 2024-05-15T07:08:08.127Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = clients/ddm-admin-client/src/lib.rs:164
|
199 | 2024-05-15T07:08:08.159Z | INFO | SledAgent: using '[AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]' as data links for xde driver
file = illumos-utils/src/opte/illumos.rs:90
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
200 | 2024-05-15T07:08:08.160Z | INFO | SledAgent (dropshot): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196
local_addr = [fd00:1122:3344:101::1]:55688
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
201 | 2024-05-15T07:08:08.160Z | INFO | SledAgent (internal-dns-resolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
202 | 2024-05-15T07:08:08.160Z | INFO | SledAgent (producer-server): starting oximeter metric producer server
address = [fd00:1122:3344:101::1]:55688
file = oximeter/producer/src/lib.rs:284
interval = 30s
producer_id = acb2a796-8e9f-4811-b601-726a12d79b0b
route = //acb2a796-8e9f-4811-b601-726a12d79b0b
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
203 | 2024-05-15T07:08:08.167Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 210.883973ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:393
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
204 | 2024-05-15T07:08:08.208Z | INFO | SledAgent (VmmReservoirManager): 50% of 102952026112 physical ram = 51476013056 bytes)
file = sled-agent/src/vmm_reservoir.rs:267
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
205 | 2024-05-15T07:08:08.208Z | INFO | SledAgent (VmmReservoirManager): Setting reservoir size to 49090 MiB bytes
file = sled-agent/src/vmm_reservoir.rs:275
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
206 | 2024-05-15T07:08:08.208Z | INFO | SledAgent (ServiceManager): sled agent started
file = sled-agent/src/services.rs:993
underlay_address = fd00:1122:3344:101::1
|
207 | 2024-05-15T07:08:08.208Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
208 | 2024-05-15T07:08:08.209Z | INFO | SledAgent (dropshot (SledAgent)): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196
local_addr = [fd00:1122:3344:101::1]:12345
|
209 | 2024-05-15T07:08:08.209Z | INFO | SledAgent (SledAgentMain): Sled Agent loaded; recording configuration
file = sled-agent/src/bootstrap/server.rs:430
|
210 | 2024-05-15T07:08:08.209Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
211 | 2024-05-15T07:08:08.209Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:199
|
212 | 2024-05-15T07:08:08.210Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:199
|
213 | 2024-05-15T07:08:08.210Z | INFO | SledAgent (HardwareMonitor): Sled Agent Started
file = sled-agent/src/hardware_monitor.rs:138
|
214 | 2024-05-15T07:08:08.210Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
disks = {}
file = sled-agent/src/hardware_monitor.rs:262
underlay_network_info = Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [V4(Ipv4Network { addr: 10.151.2.174, prefix: 24 })], switch: Switch0, port: "qsfp0", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))
|
215 | 2024-05-15T07:08:08.210Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3617
|
216 | 2024-05-15T07:08:08.210Z | INFO | SledAgent (SledAgentSprocketsServer): Connection closed
file = sled-agent/src/bootstrap/sprockets_server.rs:73
remote_addr = [fdb0:18c0:4d0d:a02a::1]:53471
|
217 | 2024-05-15T07:08:08.388Z | INFO | SledAgent (BootstrapAgentRssHandler): Peer agent initialized
file = sled-agent/src/bootstrap/rss_handle.rs:108
peer = [fdb0:18c0:4d0d:a02a::1]:12346
|
218 | 2024-05-15T07:08:08.388Z | INFO | SledAgent (BootstrapAgentRssHandler): Initialized sled agent
file = sled-agent/src/bootstrap/rss_handle.rs:232
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
219 | 2024-05-15T07:08:08.388Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
220 | 2024-05-15T07:08:08.389Z | INFO | SledAgent (RSS): No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/rss-service-plan-v3.json
file = common/src/ledger.rs:188
|
221 | 2024-05-15T07:08:08.389Z | INFO | SledAgent (RSS): No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/rss-service-plan-v3.json
file = common/src/ledger.rs:188
|
222 | 2024-05-15T07:08:08.389Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
223 | 2024-05-15T07:08:08.389Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
224 | 2024-05-15T07:08:08.390Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
225 | 2024-05-15T07:08:08.390Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 464.359786ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:393
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
226 | 2024-05-15T07:08:08.421Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:42554
|
227 | 2024-05-15T07:08:08.421Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
228 | 2024-05-15T07:08:08.421Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 164
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:42554
req_id = 54c00772-1052-4c33-8ab3-066a02009ede
response_code = 200
uri = /inventory
|
229 | 2024-05-15T07:08:08.453Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:37703
|
230 | 2024-05-15T07:08:08.453Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 33
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:37703
req_id = 56c373a4-1220-4b81-a0a0-9544056e8f67
response_code = 200
uri = /sled-role
|
231 | 2024-05-15T07:08:08.454Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
232 | 2024-05-15T07:08:08.454Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.rss-service-plan-v3.json.tmp
file = common/src/ledger.rs:199
|
233 | 2024-05-15T07:08:08.455Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.rss-service-plan-v3.json.tmp
file = common/src/ledger.rs:199
|
234 | 2024-05-15T07:08:08.455Z | INFO | SledAgent (RSS): Service plan written to storage
file = sled-agent/src/rack_setup/plan/service.rs:815
|
235 | 2024-05-15T07:08:08.486Z | INFO | SledAgent (RSS): attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: bf2a0dce-f659-4a90-aa0b-dd0a03b87954, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, pool_id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool) }, OmicronPhysicalDiskConfig { id: 3da5cdb7-031b-441d-bcba-8297bc53c14f, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, pool_id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool) }, OmicronPhysicalDiskConfig { id: 2da42829-b77d-42b5-8212-48a387442b7c, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, pool_id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool) }, OmicronPhysicalDiskConfig { id: d27566ee-ab55-4b03-8f38-07e4c4675371, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, pool_id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool) }, OmicronPhysicalDiskConfig { id: 6c962dd2-23d0-4373-b1a1-9279cf19ae6d, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, pool_id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool) }, OmicronPhysicalDiskConfig { id: a3e4dcc9-7f51-462f-af2f-8c74c5b2c173, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, pool_id: 5da68ecb-08f6-4895-925f-903c81000441 (zpool) }, OmicronPhysicalDiskConfig { id: 4c36d026-3af1-492b-8432-acefaab135a6, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }, pool_id: 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool) }, OmicronPhysicalDiskConfig { id: 2150da11-52cd-42fe-9d41-aae1a3441958, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }, pool_id: 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool) }, OmicronPhysicalDiskConfig { id: 575ec239-a350-4c7d-8ea3-b2a988f60446, identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }, pool_id: bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool) }], generation: Generation(1) }
file = sled-agent/src/rack_setup/service.rs:347
sled_address = [fd00:1122:3344:101::1]:12345
|
236 | 2024-05-15T07:08:08.486Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:52314
|
237 | 2024-05-15T07:08:08.486Z | INFO | SledAgent (StorageManager): Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, id: bf2a0dce-f659-4a90-aa0b-dd0a03b87954, pool_id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, id: 3da5cdb7-031b-441d-bcba-8297bc53c14f, pool_id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, id: 2da42829-b77d-42b5-8212-48a387442b7c, pool_id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, id: d27566ee-ab55-4b03-8f38-07e4c4675371, pool_id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, id: 6c962dd2-23d0-4373-b1a1-9279cf19ae6d, pool_id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, id: a3e4dcc9-7f51-462f-af2f-8c74c5b2c173, pool_id: 5da68ecb-08f6-4895-925f-903c81000441 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }, id: 4c36d026-3af1-492b-8432-acefaab135a6, pool_id: 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }, id: 2150da11-52cd-42fe-9d41-aae1a3441958, pool_id: 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }, id: 575ec239-a350-4c7d-8ea3-b2a988f60446, pool_id: bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool) }] }, tx: ... }
file = sled-storage/src/manager.rs:381
|
238 | 2024-05-15T07:08:08.486Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = omicron_physical_disks_ensure
|
239 | 2024-05-15T07:08:08.486Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = omicron_physical_disks_ensure
|
240 | 2024-05-15T07:08:08.486Z | INFO | SledAgent (StorageManager): No previously-stored 'requested disks', creating new ledger
file = sled-storage/src/manager.rs:681
request = omicron_physical_disks_ensure
|
241 | 2024-05-15T07:08:08.486Z | INFO | SledAgent (StorageResources): Synchronizing disk managment
file = sled-storage/src/resources.rs:339
|
242 | 2024-05-15T07:08:08.486Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
243 | 2024-05-15T07:08:08.486Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
244 | 2024-05-15T07:08:08.486Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:112
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
path = /scratch/u2_0.vdev
|
245 | 2024-05-15T07:08:08.580Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev
file = sled-hardware/src/disk.rs:304
|
246 | 2024-05-15T07:08:08.580Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool)
|
247 | 2024-05-15T07:08:08.947Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 877.607412ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:393
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
248 | 2024-05-15T07:08:09.212Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
249 | 2024-05-15T07:08:09.590Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }
|
250 | 2024-05-15T07:08:09.765Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
251 | 2024-05-15T07:08:09.765Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
252 | 2024-05-15T07:08:09.765Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
253 | 2024-05-15T07:08:09.765Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
254 | 2024-05-15T07:08:09.765Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key")
|
255 | 2024-05-15T07:08:09.765Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key")
|
256 | 2024-05-15T07:08:09.765Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
257 | 2024-05-15T07:08:09.950Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 2.903407045s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:393
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
258 | 2024-05-15T07:08:10.325Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_switch
|
259 | 2024-05-15T07:08:10.519Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
260 | 2024-05-15T07:08:10.736Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
261 | 2024-05-15T07:08:10.736Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
262 | 2024-05-15T07:08:10.948Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_4e6ef30f-a7d9-4acc-99aa-59ae1084fa14/crypt/zone
file = sled-storage/src/dataset.rs:395
|
263 | 2024-05-15T07:08:11.135Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 61.742857ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
264 | 2024-05-15T07:08:11.669Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 112.265741ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
265 | 2024-05-15T07:08:12.281Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
266 | 2024-05-15T07:08:12.281Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
267 | 2024-05-15T07:08:12.281Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 114.940381ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
268 | 2024-05-15T07:08:12.785Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 482.068348ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
269 | 2024-05-15T07:08:12.964Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 3.84884602s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:393
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
270 | 2024-05-15T07:08:13.614Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_4e6ef30f-a7d9-4acc-99aa-59ae1084fa14
file = sled-storage/src/dataset.rs:480
|
271 | 2024-05-15T07:08:13.847Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
272 | 2024-05-15T07:08:13.847Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 1.166310241s
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
273 | 2024-05-15T07:08:13.853Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
274 | 2024-05-15T07:08:13.853Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
275 | 2024-05-15T07:08:13.854Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
276 | 2024-05-15T07:08:13.854Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
277 | 2024-05-15T07:08:13.854Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:112
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
path = /scratch/u2_1.vdev
|
278 | 2024-05-15T07:08:14.091Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev
file = sled-hardware/src/disk.rs:304
|
279 | 2024-05-15T07:08:14.091Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool)
|
280 | 2024-05-15T07:08:15.280Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }
|
281 | 2024-05-15T07:08:15.510Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
282 | 2024-05-15T07:08:15.510Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
283 | 2024-05-15T07:08:15.510Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
284 | 2024-05-15T07:08:15.510Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
285 | 2024-05-15T07:08:15.510Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key")
|
286 | 2024-05-15T07:08:15.510Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key")
|
287 | 2024-05-15T07:08:15.510Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
288 | 2024-05-15T07:08:16.281Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
289 | 2024-05-15T07:08:16.764Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
290 | 2024-05-15T07:08:16.997Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 4.765630932s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:393
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
291 | 2024-05-15T07:08:17.238Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
292 | 2024-05-15T07:08:17.495Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
293 | 2024-05-15T07:08:17.754Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_12d428b9-eedd-40c0-8928-1d92a73f181d/crypt/zone
file = sled-storage/src/dataset.rs:395
|
294 | 2024-05-15T07:08:17.986Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
295 | 2024-05-15T07:08:17.986Z | INFO | SledAgent (ServiceManager): Ensuring tfportrear0_0/ll exists in zone
file = sled-agent/src/services.rs:2303
|
296 | 2024-05-15T07:08:18.445Z | INFO | SledAgent (VmmReservoirManager): Finished setting reservoir size to 49090 MiB bytes
file = sled-agent/src/vmm_reservoir.rs:279
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
297 | 2024-05-15T07:08:18.498Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
298 | 2024-05-15T07:08:18.563Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_12d428b9-eedd-40c0-8928-1d92a73f181d
file = sled-storage/src/dataset.rs:480
|
299 | 2024-05-15T07:08:18.571Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
300 | 2024-05-15T07:08:18.572Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
301 | 2024-05-15T07:08:18.572Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
302 | 2024-05-15T07:08:18.572Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
303 | 2024-05-15T07:08:18.572Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:112
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
path = /scratch/u2_2.vdev
|
304 | 2024-05-15T07:08:18.582Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev
file = sled-hardware/src/disk.rs:304
|
305 | 2024-05-15T07:08:18.582Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool)
|
306 | 2024-05-15T07:08:18.678Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }
|
307 | 2024-05-15T07:08:18.686Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
308 | 2024-05-15T07:08:18.686Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
309 | 2024-05-15T07:08:18.686Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
310 | 2024-05-15T07:08:18.686Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
311 | 2024-05-15T07:08:18.686Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key")
|
312 | 2024-05-15T07:08:18.686Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key")
|
313 | 2024-05-15T07:08:18.686Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
314 | 2024-05-15T07:08:18.721Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
315 | 2024-05-15T07:08:18.730Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_08d4507c-e72e-4afd-9f03-5e28f4ad0909/crypt/zone
file = sled-storage/src/dataset.rs:395
|
316 | 2024-05-15T07:08:18.855Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_08d4507c-e72e-4afd-9f03-5e28f4ad0909
file = sled-storage/src/dataset.rs:480
|
317 | 2024-05-15T07:08:18.863Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
318 | 2024-05-15T07:08:18.863Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
319 | 2024-05-15T07:08:18.863Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
320 | 2024-05-15T07:08:18.863Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
321 | 2024-05-15T07:08:18.863Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:112
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
path = /scratch/u2_3.vdev
|
322 | 2024-05-15T07:08:18.873Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev
file = sled-hardware/src/disk.rs:304
|
323 | 2024-05-15T07:08:18.873Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool)
|
324 | 2024-05-15T07:08:18.966Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }
|
325 | 2024-05-15T07:08:18.974Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
326 | 2024-05-15T07:08:18.974Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
327 | 2024-05-15T07:08:18.974Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
328 | 2024-05-15T07:08:18.974Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
329 | 2024-05-15T07:08:18.974Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key")
|
330 | 2024-05-15T07:08:18.974Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key")
|
331 | 2024-05-15T07:08:18.974Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
332 | 2024-05-15T07:08:19.011Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
333 | 2024-05-15T07:08:19.019Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_7ba4af02-bf8f-47c6-af79-d69fdad5d4bf/crypt/zone
file = sled-storage/src/dataset.rs:395
|
334 | 2024-05-15T07:08:19.145Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_7ba4af02-bf8f-47c6-af79-d69fdad5d4bf
file = sled-storage/src/dataset.rs:480
|
335 | 2024-05-15T07:08:19.153Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
336 | 2024-05-15T07:08:19.153Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
337 | 2024-05-15T07:08:19.153Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
338 | 2024-05-15T07:08:19.153Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
339 | 2024-05-15T07:08:19.153Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:112
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
path = /scratch/u2_4.vdev
|
340 | 2024-05-15T07:08:19.163Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev
file = sled-hardware/src/disk.rs:304
|
341 | 2024-05-15T07:08:19.163Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool)
|
342 | 2024-05-15T07:08:19.260Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }
|
343 | 2024-05-15T07:08:19.268Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
344 | 2024-05-15T07:08:19.268Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
345 | 2024-05-15T07:08:19.268Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
346 | 2024-05-15T07:08:19.268Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
347 | 2024-05-15T07:08:19.268Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key")
|
348 | 2024-05-15T07:08:19.268Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key")
|
349 | 2024-05-15T07:08:19.268Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
350 | 2024-05-15T07:08:19.304Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
351 | 2024-05-15T07:08:19.311Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_62a187a2-0706-420e-a881-83ff781ad6c1/crypt/zone
file = sled-storage/src/dataset.rs:395
|
352 | 2024-05-15T07:08:19.436Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_62a187a2-0706-420e-a881-83ff781ad6c1
file = sled-storage/src/dataset.rs:480
|
353 | 2024-05-15T07:08:19.445Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
354 | 2024-05-15T07:08:19.445Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
355 | 2024-05-15T07:08:19.445Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
356 | 2024-05-15T07:08:19.445Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
357 | 2024-05-15T07:08:19.445Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:112
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
path = /scratch/u2_5.vdev
|
358 | 2024-05-15T07:08:19.455Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev
file = sled-hardware/src/disk.rs:304
|
359 | 2024-05-15T07:08:19.455Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 5da68ecb-08f6-4895-925f-903c81000441 (zpool)
|
360 | 2024-05-15T07:08:19.555Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 5da68ecb-08f6-4895-925f-903c81000441 (zpool), kind: External }
|
361 | 2024-05-15T07:08:19.562Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
362 | 2024-05-15T07:08:19.562Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
363 | 2024-05-15T07:08:19.562Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
364 | 2024-05-15T07:08:19.562Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
365 | 2024-05-15T07:08:19.562Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key")
|
366 | 2024-05-15T07:08:19.562Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key")
|
367 | 2024-05-15T07:08:19.562Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
368 | 2024-05-15T07:08:19.567Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
369 | 2024-05-15T07:08:19.598Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
370 | 2024-05-15T07:08:19.606Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_5da68ecb-08f6-4895-925f-903c81000441/crypt/zone
file = sled-storage/src/dataset.rs:395
|
371 | 2024-05-15T07:08:19.730Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_5da68ecb-08f6-4895-925f-903c81000441
file = sled-storage/src/dataset.rs:480
|
372 | 2024-05-15T07:08:19.738Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
373 | 2024-05-15T07:08:19.738Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
374 | 2024-05-15T07:08:19.738Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
375 | 2024-05-15T07:08:19.738Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
376 | 2024-05-15T07:08:19.738Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:112
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
path = /scratch/u2_6.vdev
|
377 | 2024-05-15T07:08:19.748Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev
file = sled-hardware/src/disk.rs:304
|
378 | 2024-05-15T07:08:19.748Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool)
|
379 | 2024-05-15T07:08:19.843Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool), kind: External }
|
380 | 2024-05-15T07:08:19.851Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
381 | 2024-05-15T07:08:19.851Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
382 | 2024-05-15T07:08:19.851Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
383 | 2024-05-15T07:08:19.851Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
384 | 2024-05-15T07:08:19.851Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key")
|
385 | 2024-05-15T07:08:19.851Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key")
|
386 | 2024-05-15T07:08:19.851Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
387 | 2024-05-15T07:08:19.854Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
388 | 2024-05-15T07:08:19.888Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
389 | 2024-05-15T07:08:19.897Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_3b36f3f1-29d0-4d11-8b6b-ae2a89620d43/crypt/zone
file = sled-storage/src/dataset.rs:395
|
390 | 2024-05-15T07:08:20.029Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_3b36f3f1-29d0-4d11-8b6b-ae2a89620d43
file = sled-storage/src/dataset.rs:480
|
391 | 2024-05-15T07:08:20.038Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
392 | 2024-05-15T07:08:20.038Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
393 | 2024-05-15T07:08:20.038Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
394 | 2024-05-15T07:08:20.038Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
395 | 2024-05-15T07:08:20.038Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:112
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
path = /scratch/u2_7.vdev
|
396 | 2024-05-15T07:08:20.048Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev
file = sled-hardware/src/disk.rs:304
|
397 | 2024-05-15T07:08:20.048Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool)
|
398 | 2024-05-15T07:08:20.141Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool), kind: External }
|
399 | 2024-05-15T07:08:20.147Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
400 | 2024-05-15T07:08:20.148Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
401 | 2024-05-15T07:08:20.148Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
402 | 2024-05-15T07:08:20.148Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
403 | 2024-05-15T07:08:20.148Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key")
|
404 | 2024-05-15T07:08:20.148Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key")
|
405 | 2024-05-15T07:08:20.148Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
406 | 2024-05-15T07:08:20.185Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
407 | 2024-05-15T07:08:20.193Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_2780ea55-1799-4c3e-916a-9024b9586eaf/crypt/zone
file = sled-storage/src/dataset.rs:395
|
408 | 2024-05-15T07:08:20.314Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_2780ea55-1799-4c3e-916a-9024b9586eaf
file = sled-storage/src/dataset.rs:480
|
409 | 2024-05-15T07:08:20.322Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
410 | 2024-05-15T07:08:20.322Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
411 | 2024-05-15T07:08:20.322Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:379
|
412 | 2024-05-15T07:08:20.322Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:456
|
413 | 2024-05-15T07:08:20.322Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:112
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
path = /scratch/u2_8.vdev
|
414 | 2024-05-15T07:08:20.331Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev
file = sled-hardware/src/disk.rs:304
|
415 | 2024-05-15T07:08:20.331Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:310
id = bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool)
|
416 | 2024-05-15T07:08:20.426Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:288
zpool = ZpoolName { id: bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool), kind: External }
|
417 | 2024-05-15T07:08:20.433Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
file = sled-storage/src/dataset.rs:332
|
418 | 2024-05-15T07:08:20.433Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:334
|
419 | 2024-05-15T07:08:20.433Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:338
|
420 | 2024-05-15T07:08:20.433Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
epoch = 0
file = sled-storage/src/dataset.rs:340
|
421 | 2024-05-15T07:08:20.433Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key")
|
422 | 2024-05-15T07:08:20.433Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key")
|
423 | 2024-05-15T07:08:20.433Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:352
|
424 | 2024-05-15T07:08:20.470Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
425 | 2024-05-15T07:08:20.478Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_bae019e4-4e95-466e-aa8a-f660660e26c1/crypt/zone
file = sled-storage/src/dataset.rs:395
|
426 | 2024-05-15T07:08:20.569Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
427 | 2024-05-15T07:08:20.602Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_bae019e4-4e95-466e-aa8a-f660660e26c1
file = sled-storage/src/dataset.rs:480
|
428 | 2024-05-15T07:08:20.610Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:477
|
429 | 2024-05-15T07:08:20.610Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
file = sled-storage/src/resources.rs:394
|
430 | 2024-05-15T07:08:20.610Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.omicron-physical-disks.json.tmp
file = common/src/ledger.rs:199
request = omicron_physical_disks_ensure
|
431 | 2024-05-15T07:08:20.610Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { values: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 5da68ecb-08f6-4895-925f-903c81000441 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool), kind: External } }))}, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:41
|
432 | 2024-05-15T07:08:20.610Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:549
|
433 | 2024-05-15T07:08:20.610Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
434 | 2024-05-15T07:08:20.610Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.omicron-physical-disks.json.tmp
file = common/src/ledger.rs:199
request = omicron_physical_disks_ensure
|
435 | 2024-05-15T07:08:20.611Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 12124443
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:52314
req_id = a8bd8eae-8392-415b-b726-51516711c801
response_code = 200
uri = /omicron-physical-disks
|
436 | 2024-05-15T07:08:20.641Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
437 | 2024-05-15T07:08:20.641Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:50895
|
438 | 2024-05-15T07:08:20.642Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
439 | 2024-05-15T07:08:20.642Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
440 | 2024-05-15T07:08:20.693Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
441 | 2024-05-15T07:08:20.693Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
442 | 2024-05-15T07:08:20.745Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
443 | 2024-05-15T07:08:20.745Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
444 | 2024-05-15T07:08:20.797Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
445 | 2024-05-15T07:08:20.797Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:188
|
446 | 2024-05-15T07:08:20.797Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:188
|
447 | 2024-05-15T07:08:20.797Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
448 | 2024-05-15T07:08:20.842Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
449 | 2024-05-15T07:08:20.842Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
450 | 2024-05-15T07:08:20.842Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
451 | 2024-05-15T07:08:20.896Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982
file = illumos-utils/src/zone.rs:315
|
452 | 2024-05-15T07:08:20.900Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c
file = illumos-utils/src/zone.rs:315
|
453 | 2024-05-15T07:08:20.904Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62
file = illumos-utils/src/zone.rs:315
|
454 | 2024-05-15T07:08:20.917Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982
file = illumos-utils/src/zone.rs:354
|
455 | 2024-05-15T07:08:20.921Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c
file = illumos-utils/src/zone.rs:354
|
456 | 2024-05-15T07:08:20.925Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62
file = illumos-utils/src/zone.rs:354
|
457 | 2024-05-15T07:08:21.571Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/probe_manager.rs:169
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
458 | 2024-05-15T07:08:21.764Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 15.454121794s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:393
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
459 | 2024-05-15T07:08:21.853Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/nexus.rs:619
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
460 | 2024-05-15T07:08:22.442Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService1"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:2::2"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:2::1"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:2::1]:5353"/>
<propval type="astring" name="dns_address" value="[fd00:1122:3344:2::1]:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
461 | 2024-05-15T07:08:22.442Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }
|
462 | 2024-05-15T07:08:22.442Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982
|
463 | 2024-05-15T07:08:22.929Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService2"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:3::2"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:3::1"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:3::1]:5353"/>
<propval type="astring" name="dns_address" value="[fd00:1122:3344:3::1]:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
464 | 2024-05-15T07:08:23.416Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }
|
465 | 2024-05-15T07:08:23.417Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService3"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:1::2"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:1::1"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:1::1]:5353"/>
<propval type="astring" name="dns_address" value="[fd00:1122:3344:1::1]:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
466 | 2024-05-15T07:08:23.417Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:87
prefix = Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }
|
467 | 2024-05-15T07:08:23.425Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982") failed: Property not found. retry in 69.341009ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982
|
468 | 2024-05-15T07:08:23.425Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c
|
469 | 2024-05-15T07:08:23.429Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62
|
470 | 2024-05-15T07:08:23.504Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982") failed: Property not found. retry in 138.27505ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982
|
471 | 2024-05-15T07:08:23.549Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c") failed: Property not found. retry in 66.1532ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c
|
472 | 2024-05-15T07:08:23.555Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62") failed: Property not found. retry in 34.756274ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62
|
473 | 2024-05-15T07:08:23.597Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62") failed: Property not found. retry in 134.171434ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62
|
474 | 2024-05-15T07:08:23.702Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c") failed: Property not found. retry in 120.441799ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c
|
475 | 2024-05-15T07:08:23.759Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982") failed: Property not found. retry in 273.14767ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982
|
476 | 2024-05-15T07:08:23.877Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62") failed: Property not found. retry in 208.055766ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62
|
477 | 2024-05-15T07:08:23.898Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c") failed: Property not found. retry in 109.485985ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c
|
478 | 2024-05-15T07:08:24.027Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c") failed: Property not found. retry in 571.647691ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c
|
479 | 2024-05-15T07:08:24.042Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982") failed: Property not found. retry in 300.448797ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982
|
480 | 2024-05-15T07:08:24.096Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62") failed: Property not found. retry in 480.014683ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62
|
481 | 2024-05-15T07:08:24.350Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982") failed: Property not found. retry in 506.309752ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982
|
482 | 2024-05-15T07:08:24.586Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62") failed: Property not found. retry in 952.571801ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62
|
483 | 2024-05-15T07:08:24.608Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c") failed: Property not found. retry in 1.160549834s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c
|
484 | 2024-05-15T07:08:24.869Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982") failed: Property not found. retry in 1.485934287s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982
|
485 | 2024-05-15T07:08:25.550Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62") failed: Property not found. retry in 886.927132ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62
|
486 | 2024-05-15T07:08:25.778Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c") failed: Property not found. retry in 1.17524673s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c
|
487 | 2024-05-15T07:08:26.021Z | INFO | SledAgent (ServiceManager): Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone
file = sled-agent/src/services.rs:2320
|
488 | 2024-05-15T07:08:26.021Z | INFO | SledAgent (ServiceManager): Adding bootstrap address
file = illumos-utils/src/running_zone.rs:626
zone = oxz_switch
|
489 | 2024-05-15T07:08:26.364Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982") failed: Property not found. retry in 754.976855ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982
|
490 | 2024-05-15T07:08:26.445Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62") failed: Property not found. retry in 580.641515ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62
|
491 | 2024-05-15T07:08:26.568Z | INFO | SledAgent (ServiceManager): Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fedf:4abf
file = sled-agent/src/services.rs:2327
|
492 | 2024-05-15T07:08:26.573Z | INFO | SledAgent (ServiceManager): Zone not using gateway (even though underlay is up)
file = sled-agent/src/services.rs:2388
|
493 | 2024-05-15T07:08:26.691Z | INFO | SledAgent (ServiceManager): Setting up dendrite service
file = sled-agent/src/services.rs:2573
|
494 | 2024-05-15T07:08:26.931Z | INFO | SledAgent (ServiceManager): Setting up lldpd service
file = sled-agent/src/services.rs:2737
|
495 | 2024-05-15T07:08:26.964Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c") failed: Property not found. retry in 1.058980197s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c
|
496 | 2024-05-15T07:08:27.034Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62") failed: Property not found. retry in 1.194010768s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62
|
497 | 2024-05-15T07:08:27.124Z | INFO | SledAgent (ServiceManager): Setting up MGS service
file = sled-agent/src/services.rs:2452
|
498 | 2024-05-15T07:08:27.128Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982") failed: Property not found. retry in 696.935222ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982
|
499 | 2024-05-15T07:08:27.427Z | INFO | SledAgent (ServiceManager): Setting up wicketd service
file = sled-agent/src/services.rs:2487
|
500 | 2024-05-15T07:08:27.647Z | INFO | SledAgent (ServiceManager): Setting up mgd service
file = sled-agent/src/services.rs:2787
|
501 | 2024-05-15T07:08:27.831Z | INFO | SledAgent (ServiceManager): Setting up mg-ddm service
file = sled-agent/src/services.rs:2814
|
502 | 2024-05-15T07:08:27.835Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982") failed: Property not found. retry in 1.129063318s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982
|
503 | 2024-05-15T07:08:28.034Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c") failed: Property not found. retry in 841.067879ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c
|
504 | 2024-05-15T07:08:28.048Z | INFO | SledAgent (ServiceManager): Setting up tfport service
file = sled-agent/src/services.rs:2681
|
505 | 2024-05-15T07:08:28.235Z | INFO | SledAgent (ServiceManager): Setting up Simulated SP service
file = sled-agent/src/services.rs:2481
|
506 | 2024-05-15T07:08:28.238Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62") failed: Property not found. retry in 1.113305531s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62
|
507 | 2024-05-15T07:08:28.245Z | INFO | SledAgent (ServiceManager): Re-enabling running switch zone (new address)
file = sled-agent/src/services.rs:3873
new = [fd00:1122:3344:101::2, ::1]
old = [::1]
|
508 | 2024-05-15T07:08:28.245Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists
file = sled-agent/src/services.rs:3889
|
509 | 2024-05-15T07:08:28.246Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:609
zone = oxz_switch
|
510 | 2024-05-15T07:08:28.801Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists - OK
file = sled-agent/src/services.rs:3897
|
511 | 2024-05-15T07:08:28.808Z | INFO | SledAgent (ServiceManager): configuring dendrite service
file = sled-agent/src/services.rs:3953
|
512 | 2024-05-15T07:08:28.967Z | INFO | SledAgent (ServiceManager): configuring lldp service
file = sled-agent/src/services.rs:4015
|
513 | 2024-05-15T07:08:29.067Z | INFO | SledAgent (ServiceManager): configuring wicketd
file = sled-agent/src/services.rs:3996
rack_subnet = fd00:1122:3344::
|
514 | 2024-05-15T07:08:29.088Z | INFO | SledAgent (ServiceManager): configuring mgd service
file = sled-agent/src/services.rs:4043
|
515 | 2024-05-15T07:08:29.184Z | INFO | SledAgent (ServiceManager): configuring mg-ddm service
file = sled-agent/src/services.rs:4070
|
516 | 2024-05-15T07:08:29.308Z | INFO | SledAgent (ServiceManager): Determining physical location of our switch zone at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:365
|
517 | 2024-05-15T07:08:29.340Z | INFO | SledAgent (ServiceManager): Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:413
|
518 | 2024-05-15T07:08:29.370Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982
|
519 | 2024-05-15T07:08:29.370Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c
|
520 | 2024-05-15T07:08:29.370Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62
|
521 | 2024-05-15T07:08:29.370Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
522 | 2024-05-15T07:08:29.371Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
523 | 2024-05-15T07:08:29.371Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 8729531
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50895
req_id = 8908c98c-00ab-42cf-93e4-be0163903f21
response_code = 204
uri = /omicron-zones
|
524 | 2024-05-15T07:08:29.373Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:1::1]:5353
file = sled-agent/src/rack_setup/service.rs:548
|
525 | 2024-05-15T07:08:29.379Z | INFO | SledAgent (ServiceManager): Building Port Configuration
file = sled-agent/src/bootstrap/early_networking.rs:651
|
526 | 2024-05-15T07:08:29.379Z | INFO | SledAgent (ServiceManager): Checking dendrite uptime
file = sled-agent/src/bootstrap/early_networking.rs:689
|
527 | 2024-05-15T07:08:29.379Z | INFO | SledAgent (ServiceManager): Dendrite online
file = sled-agent/src/bootstrap/early_networking.rs:692
uptime = 3
|
528 | 2024-05-15T07:08:29.379Z | INFO | SledAgent (ServiceManager): Configuring default uplink on switch
config = PortSettings {\n links: {\n "0": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}
file = sled-agent/src/bootstrap/early_networking.rs:435
|
529 | 2024-05-15T07:08:29.426Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:2::1]:5353
file = sled-agent/src/rack_setup/service.rs:548
|
530 | 2024-05-15T07:08:29.462Z | INFO | SledAgent (StorageManager): Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }
file = sled-storage/src/manager.rs:381
|
531 | 2024-05-15T07:08:29.462Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }
|
532 | 2024-05-15T07:08:29.462Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-M2" }
|
533 | 2024-05-15T07:08:29.462Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
|
534 | 2024-05-15T07:08:29.462Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-0", model: "synthetic-model-U2" }
|
535 | 2024-05-15T07:08:29.462Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }
|
536 | 2024-05-15T07:08:29.462Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-M2" }
|
537 | 2024-05-15T07:08:29.462Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
|
538 | 2024-05-15T07:08:29.462Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-1", model: "synthetic-model-U2" }
|
539 | 2024-05-15T07:08:29.462Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
|
540 | 2024-05-15T07:08:29.462Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-2", model: "synthetic-model-U2" }
|
541 | 2024-05-15T07:08:29.462Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
|
542 | 2024-05-15T07:08:29.462Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-3", model: "synthetic-model-U2" }
|
543 | 2024-05-15T07:08:29.462Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
|
544 | 2024-05-15T07:08:29.463Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-4", model: "synthetic-model-U2" }
|
545 | 2024-05-15T07:08:29.463Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
|
546 | 2024-05-15T07:08:29.463Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-5", model: "synthetic-model-U2" }
|
547 | 2024-05-15T07:08:29.463Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
|
548 | 2024-05-15T07:08:29.463Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-6", model: "synthetic-model-U2" }
|
549 | 2024-05-15T07:08:29.463Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
|
550 | 2024-05-15T07:08:29.463Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-7", model: "synthetic-model-U2" }
|
551 | 2024-05-15T07:08:29.463Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:532
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
|
552 | 2024-05-15T07:08:29.463Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:553
identity = DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-8", model: "synthetic-model-U2" }
|
553 | 2024-05-15T07:08:29.490Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:3::1]:5353
file = sled-agent/src/rack_setup/service.rs:548
|
554 | 2024-05-15T07:08:29.537Z | INFO | SledAgent (RSS): Configured all DNS servers
file = sled-agent/src/rack_setup/service.rs:584
|
555 | 2024-05-15T07:08:29.537Z | INFO | SledAgent (RSS): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:259
|
556 | 2024-05-15T07:08:29.569Z | INFO | SledAgent (RSS): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:276
|
557 | 2024-05-15T07:08:29.569Z | INFO | SledAgent (RSS): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:301
|
558 | 2024-05-15T07:08:29.569Z | INFO | SledAgent (RSS): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:193
|
559 | 2024-05-15T07:08:29.598Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
560 | 2024-05-15T07:08:29.598Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:49467
|
561 | 2024-05-15T07:08:29.598Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
562 | 2024-05-15T07:08:29.598Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
563 | 2024-05-15T07:08:29.619Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
564 | 2024-05-15T07:08:29.619Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
565 | 2024-05-15T07:08:29.636Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
566 | 2024-05-15T07:08:29.636Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
567 | 2024-05-15T07:08:29.655Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
568 | 2024-05-15T07:08:29.655Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
569 | 2024-05-15T07:08:29.655Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
570 | 2024-05-15T07:08:29.655Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
571 | 2024-05-15T07:08:29.665Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:259
|
572 | 2024-05-15T07:08:29.697Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:276
|
573 | 2024-05-15T07:08:29.697Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:301
|
574 | 2024-05-15T07:08:29.698Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:193
|
575 | 2024-05-15T07:08:29.742Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte0", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 138, 18]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: "vopte0" } }
|
576 | 2024-05-15T07:08:29.742Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = ntp
|
577 | 2024-05-15T07:08:29.742Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
578 | 2024-05-15T07:08:29.742Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
579 | 2024-05-15T07:08:29.743Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
580 | 2024-05-15T07:08:29.744Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
581 | 2024-05-15T07:08:29.766Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687
file = illumos-utils/src/zone.rs:315
|
582 | 2024-05-15T07:08:29.786Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687
file = illumos-utils/src/zone.rs:354
|
583 | 2024-05-15T07:08:30.416Z | INFO | SledAgent (ServiceManager): Profile for oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService4"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::1b"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/chrony-setup">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="allow" value="fd00:1122:3344:100::/56"/>
<propval type="boolean" name="boundary" value="true"/>
<propval type="astring" name="server" value="0.pool.ntp.org"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="1.1.1.1" />
<value_node value="9.9.9.9" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/ntp">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte0"/>
<propval type="astring" name="gateway" value="172.30.3.1"/>
<propval type="astring" name="ip" value="172.30.3.5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
584 | 2024-05-15T07:08:30.416Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687
|
585 | 2024-05-15T07:08:30.557Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687") failed: Property not found. retry in 30.606696ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687
|
586 | 2024-05-15T07:08:30.595Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687") failed: Property not found. retry in 87.265972ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687
|
587 | 2024-05-15T07:08:30.695Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687") failed: Property not found. retry in 126.830481ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687
|
588 | 2024-05-15T07:08:30.833Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687") failed: Property not found. retry in 332.146718ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687
|
589 | 2024-05-15T07:08:31.175Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687") failed: Property not found. retry in 1.031646728s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687
|
590 | 2024-05-15T07:08:32.218Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687") failed: Property not found. retry in 894.92526ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687
|
591 | 2024-05-15T07:08:32.731Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:61118
|
592 | 2024-05-15T07:08:32.732Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 46
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:61118
req_id = b6a26b1f-9489-4234-9b3f-ab52232fde61
response_code = 200
uri = /baseboard
|
593 | 2024-05-15T07:08:33.123Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687") failed: Property not found. retry in 1.341077672s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687
|
594 | 2024-05-15T07:08:34.474Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687") failed: Property not found. retry in 770.329619ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687
|
595 | 2024-05-15T07:08:35.254Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687") failed: Property not found. retry in 1.440427843s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687
|
596 | 2024-05-15T07:08:36.715Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687
|
597 | 2024-05-15T07:08:36.715Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
598 | 2024-05-15T07:08:36.715Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
599 | 2024-05-15T07:08:36.715Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 7117508
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:49467
req_id = afebd678-1fa6-4c83-8ea9-e243045880c4
response_code = 204
uri = /omicron-zones
|
600 | 2024-05-15T07:08:36.717Z | INFO | SledAgent (RSS): Waiting for rack time synchronization
file = sled-agent/src/rack_setup/service.rs:625
|
601 | 2024-05-15T07:08:36.746Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
602 | 2024-05-15T07:08:36.746Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:59323
|
603 | 2024-05-15T07:08:36.755Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3603
|
604 | 2024-05-15T07:08:36.755Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 8582
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:59323
req_id = 7f790f91-d603-4eab-99d6-12b3b699c435
response_code = 500
uri = /timesync
|
605 | 2024-05-15T07:08:36.758Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
606 | 2024-05-15T07:08:36.851Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
607 | 2024-05-15T07:08:36.851Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:35565
|
608 | 2024-05-15T07:08:40.859Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3603
|
609 | 2024-05-15T07:08:40.859Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 4007967
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:35565
req_id = cbc2d379-ccba-419d-bd67-7fe835760aca
response_code = 500
uri = /timesync
|
610 | 2024-05-15T07:08:40.864Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
611 | 2024-05-15T07:08:41.012Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
612 | 2024-05-15T07:08:41.012Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:50742
|
613 | 2024-05-15T07:08:41.021Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 9433
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:50742
req_id = 77075011-956a-497b-b697-aedd8324854b
response_code = 200
uri = /timesync
|
614 | 2024-05-15T07:08:41.024Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
615 | 2024-05-15T07:08:41.024Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
616 | 2024-05-15T07:08:41.257Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
617 | 2024-05-15T07:08:41.257Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:48949
|
618 | 2024-05-15T07:08:41.266Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 8873
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:48949
req_id = 2e190019-5674-4a9a-97dd-315d68a90b50
response_code = 200
uri = /timesync
|
619 | 2024-05-15T07:08:41.269Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
620 | 2024-05-15T07:08:41.269Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
621 | 2024-05-15T07:08:41.425Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
622 | 2024-05-15T07:08:41.425Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:51419
|
623 | 2024-05-15T07:08:41.433Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 7534
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:51419
req_id = 08566412-3cb2-4b07-a06f-12c1b915bfda
response_code = 200
uri = /timesync
|
624 | 2024-05-15T07:08:41.436Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
625 | 2024-05-15T07:08:41.436Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
626 | 2024-05-15T07:08:41.595Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
627 | 2024-05-15T07:08:41.595Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:41619
|
628 | 2024-05-15T07:08:41.602Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 7686
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:41619
req_id = a3d10a62-b7c8-4bdc-beeb-8ebc98dc75dc
response_code = 200
uri = /timesync
|
629 | 2024-05-15T07:08:41.605Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
630 | 2024-05-15T07:08:41.605Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
631 | 2024-05-15T07:08:41.956Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
632 | 2024-05-15T07:08:41.956Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:61670
|
633 | 2024-05-15T07:08:41.965Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 8441
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:61670
req_id = 02b96f3f-879f-41d4-a54f-364aa12a70dc
response_code = 200
uri = /timesync
|
634 | 2024-05-15T07:08:41.966Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
635 | 2024-05-15T07:08:41.967Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
636 | 2024-05-15T07:08:42.332Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
637 | 2024-05-15T07:08:42.332Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:53088
|
638 | 2024-05-15T07:08:42.340Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 8429
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:53088
req_id = 99df8206-c000-43e6-8edc-1e2e34ef9f1f
response_code = 200
uri = /timesync
|
639 | 2024-05-15T07:08:42.342Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
640 | 2024-05-15T07:08:42.342Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
641 | 2024-05-15T07:08:42.582Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
642 | 2024-05-15T07:08:42.583Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:46420
|
643 | 2024-05-15T07:08:42.590Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 7651
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:46420
req_id = c0986356-7ae9-4ac3-984d-67b2857842ee
response_code = 200
uri = /timesync
|
644 | 2024-05-15T07:08:42.592Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
645 | 2024-05-15T07:08:42.592Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
646 | 2024-05-15T07:08:43.150Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
647 | 2024-05-15T07:08:43.150Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:45241
|
648 | 2024-05-15T07:08:43.158Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 8094
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:45241
req_id = 6b5aa85e-4891-4447-87bc-c9eab6ffb868
response_code = 200
uri = /timesync
|
649 | 2024-05-15T07:08:43.160Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
650 | 2024-05-15T07:08:43.160Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
651 | 2024-05-15T07:08:43.889Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
652 | 2024-05-15T07:08:43.889Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:56708
|
653 | 2024-05-15T07:08:43.898Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 9021
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:56708
req_id = 19348d94-f53f-4b74-af5e-beeb5c819957
response_code = 200
uri = /timesync
|
654 | 2024-05-15T07:08:43.900Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
655 | 2024-05-15T07:08:43.900Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
656 | 2024-05-15T07:08:44.767Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
657 | 2024-05-15T07:08:44.768Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:64487
|
658 | 2024-05-15T07:08:44.776Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 7976
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:64487
req_id = 9b09a1cf-5ccc-4013-8c98-bf16360a01b0
response_code = 200
uri = /timesync
|
659 | 2024-05-15T07:08:44.777Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
660 | 2024-05-15T07:08:44.777Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:655
|
661 | 2024-05-15T07:08:45.262Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:605
|
662 | 2024-05-15T07:08:45.262Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:35742
|
663 | 2024-05-15T07:08:45.283Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 20393
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:35742
req_id = c5e3b249-bd4f-41c0-ada2-6762ca43e27e
response_code = 200
uri = /timesync
|
664 | 2024-05-15T07:08:45.286Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1163513699, ip_addr: 69.89.207.99, stratum: 2, ref_time: 1715756925.238036, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:633
|
665 | 2024-05-15T07:08:45.286Z | INFO | SledAgent (RSS): Finished setting up Internal DNS and NTP
file = sled-agent/src/rack_setup/service.rs:1216
|
666 | 2024-05-15T07:08:45.316Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
667 | 2024-05-15T07:08:45.316Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:46219
|
668 | 2024-05-15T07:08:45.316Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
669 | 2024-05-15T07:08:45.316Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
670 | 2024-05-15T07:08:45.337Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
671 | 2024-05-15T07:08:45.337Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
672 | 2024-05-15T07:08:45.356Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
673 | 2024-05-15T07:08:45.356Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
674 | 2024-05-15T07:08:45.376Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
675 | 2024-05-15T07:08:45.376Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
676 | 2024-05-15T07:08:45.436Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
677 | 2024-05-15T07:08:45.436Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
678 | 2024-05-15T07:08:45.488Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
679 | 2024-05-15T07:08:45.488Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
680 | 2024-05-15T07:08:45.540Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
681 | 2024-05-15T07:08:45.540Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
682 | 2024-05-15T07:08:45.591Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
683 | 2024-05-15T07:08:45.591Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
684 | 2024-05-15T07:08:45.640Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
685 | 2024-05-15T07:08:45.640Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
686 | 2024-05-15T07:08:45.640Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
687 | 2024-05-15T07:08:45.641Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
688 | 2024-05-15T07:08:45.673Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
689 | 2024-05-15T07:08:45.673Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
690 | 2024-05-15T07:08:45.673Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
691 | 2024-05-15T07:08:45.673Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
692 | 2024-05-15T07:08:45.673Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
693 | 2024-05-15T07:08:45.673Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 356992
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = cb805569-83a4-4e8b-800a-1a73ad54a20b
response_code = 500
uri = /omicron-zones
|
694 | 2024-05-15T07:08:45.673Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "cb805569-83a4-4e8b-800a-1a73ad54a20b", "content-length": "124", "date": "Wed, 15 May 2024 07:08:45 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "cb805569-83a4-4e8b-800a-1a73ad54a20b" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 65.832012ms
sled_address = [fd00:1122:3344:101::1]:12345
|
695 | 2024-05-15T07:08:45.741Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
696 | 2024-05-15T07:08:45.742Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
697 | 2024-05-15T07:08:45.742Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
698 | 2024-05-15T07:08:45.758Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
699 | 2024-05-15T07:08:45.758Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
700 | 2024-05-15T07:08:45.777Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
701 | 2024-05-15T07:08:45.777Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
702 | 2024-05-15T07:08:45.795Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
703 | 2024-05-15T07:08:45.795Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
704 | 2024-05-15T07:08:45.814Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
705 | 2024-05-15T07:08:45.814Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
706 | 2024-05-15T07:08:45.833Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
707 | 2024-05-15T07:08:45.833Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
708 | 2024-05-15T07:08:45.851Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
709 | 2024-05-15T07:08:45.851Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
710 | 2024-05-15T07:08:45.870Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
711 | 2024-05-15T07:08:45.870Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
712 | 2024-05-15T07:08:45.889Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
713 | 2024-05-15T07:08:45.889Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
714 | 2024-05-15T07:08:45.889Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
715 | 2024-05-15T07:08:45.889Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
716 | 2024-05-15T07:08:45.922Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
717 | 2024-05-15T07:08:45.922Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
718 | 2024-05-15T07:08:45.922Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
719 | 2024-05-15T07:08:45.922Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
720 | 2024-05-15T07:08:45.922Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
721 | 2024-05-15T07:08:45.922Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 180698
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 960bdd3e-303e-46cc-8ea6-7fb7fb14fa4f
response_code = 500
uri = /omicron-zones
|
722 | 2024-05-15T07:08:45.922Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "960bdd3e-303e-46cc-8ea6-7fb7fb14fa4f", "content-length": "124", "date": "Wed, 15 May 2024 07:08:45 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "960bdd3e-303e-46cc-8ea6-7fb7fb14fa4f" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 149.706127ms
sled_address = [fd00:1122:3344:101::1]:12345
|
723 | 2024-05-15T07:08:46.073Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
724 | 2024-05-15T07:08:46.073Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
725 | 2024-05-15T07:08:46.073Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
726 | 2024-05-15T07:08:46.094Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
727 | 2024-05-15T07:08:46.094Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
728 | 2024-05-15T07:08:46.114Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
729 | 2024-05-15T07:08:46.114Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
730 | 2024-05-15T07:08:46.134Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
731 | 2024-05-15T07:08:46.134Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
732 | 2024-05-15T07:08:46.152Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
733 | 2024-05-15T07:08:46.152Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
734 | 2024-05-15T07:08:46.170Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
735 | 2024-05-15T07:08:46.170Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
736 | 2024-05-15T07:08:46.189Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
737 | 2024-05-15T07:08:46.190Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
738 | 2024-05-15T07:08:46.208Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
739 | 2024-05-15T07:08:46.208Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
740 | 2024-05-15T07:08:46.226Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
741 | 2024-05-15T07:08:46.226Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
742 | 2024-05-15T07:08:46.226Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
743 | 2024-05-15T07:08:46.226Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
744 | 2024-05-15T07:08:46.260Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
745 | 2024-05-15T07:08:46.260Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
746 | 2024-05-15T07:08:46.260Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
747 | 2024-05-15T07:08:46.260Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
748 | 2024-05-15T07:08:46.260Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
749 | 2024-05-15T07:08:46.260Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 186852
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 35d250df-3018-433a-b9b2-f1a02f4ed89f
response_code = 500
uri = /omicron-zones
|
750 | 2024-05-15T07:08:46.260Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "35d250df-3018-433a-b9b2-f1a02f4ed89f", "content-length": "124", "date": "Wed, 15 May 2024 07:08:46 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "35d250df-3018-433a-b9b2-f1a02f4ed89f" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 203.372159ms
sled_address = [fd00:1122:3344:101::1]:12345
|
751 | 2024-05-15T07:08:46.464Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
752 | 2024-05-15T07:08:46.465Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
753 | 2024-05-15T07:08:46.465Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
754 | 2024-05-15T07:08:46.484Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
755 | 2024-05-15T07:08:46.484Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
756 | 2024-05-15T07:08:46.503Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
757 | 2024-05-15T07:08:46.503Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
758 | 2024-05-15T07:08:46.521Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
759 | 2024-05-15T07:08:46.521Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
760 | 2024-05-15T07:08:46.540Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
761 | 2024-05-15T07:08:46.540Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
762 | 2024-05-15T07:08:46.558Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
763 | 2024-05-15T07:08:46.558Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
764 | 2024-05-15T07:08:46.577Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
765 | 2024-05-15T07:08:46.577Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
766 | 2024-05-15T07:08:46.595Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
767 | 2024-05-15T07:08:46.595Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
768 | 2024-05-15T07:08:46.613Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
769 | 2024-05-15T07:08:46.613Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
770 | 2024-05-15T07:08:46.614Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
771 | 2024-05-15T07:08:46.614Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
772 | 2024-05-15T07:08:46.648Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
773 | 2024-05-15T07:08:46.648Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
774 | 2024-05-15T07:08:46.648Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
775 | 2024-05-15T07:08:46.648Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
776 | 2024-05-15T07:08:46.648Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
777 | 2024-05-15T07:08:46.648Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 183095
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = cdd554ae-a04d-4415-980e-8967950ddb2b
response_code = 500
uri = /omicron-zones
|
778 | 2024-05-15T07:08:46.648Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "cdd554ae-a04d-4415-980e-8967950ddb2b", "content-length": "124", "date": "Wed, 15 May 2024 07:08:46 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "cdd554ae-a04d-4415-980e-8967950ddb2b" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 115.118132ms
sled_address = [fd00:1122:3344:101::1]:12345
|
779 | 2024-05-15T07:08:46.764Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
780 | 2024-05-15T07:08:46.764Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
781 | 2024-05-15T07:08:46.765Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
782 | 2024-05-15T07:08:46.783Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
783 | 2024-05-15T07:08:46.783Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
784 | 2024-05-15T07:08:46.802Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
785 | 2024-05-15T07:08:46.802Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
786 | 2024-05-15T07:08:46.821Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
787 | 2024-05-15T07:08:46.821Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
788 | 2024-05-15T07:08:46.840Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
789 | 2024-05-15T07:08:46.840Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
790 | 2024-05-15T07:08:46.859Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
791 | 2024-05-15T07:08:46.859Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
792 | 2024-05-15T07:08:46.876Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
793 | 2024-05-15T07:08:46.876Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
794 | 2024-05-15T07:08:46.893Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
795 | 2024-05-15T07:08:46.893Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
796 | 2024-05-15T07:08:46.910Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
797 | 2024-05-15T07:08:46.910Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
798 | 2024-05-15T07:08:46.910Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
799 | 2024-05-15T07:08:46.910Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
800 | 2024-05-15T07:08:46.943Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
801 | 2024-05-15T07:08:46.943Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
802 | 2024-05-15T07:08:46.943Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
803 | 2024-05-15T07:08:46.943Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
804 | 2024-05-15T07:08:46.943Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
805 | 2024-05-15T07:08:46.943Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 179057
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 09293551-9be4-411a-9b90-c6e229a601c3
response_code = 500
uri = /omicron-zones
|
806 | 2024-05-15T07:08:46.944Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "09293551-9be4-411a-9b90-c6e229a601c3", "content-length": "124", "date": "Wed, 15 May 2024 07:08:46 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "09293551-9be4-411a-9b90-c6e229a601c3" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 303.627773ms
sled_address = [fd00:1122:3344:101::1]:12345
|
807 | 2024-05-15T07:08:47.249Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
808 | 2024-05-15T07:08:47.249Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
809 | 2024-05-15T07:08:47.249Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
810 | 2024-05-15T07:08:47.268Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
811 | 2024-05-15T07:08:47.268Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
812 | 2024-05-15T07:08:47.288Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
813 | 2024-05-15T07:08:47.288Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
814 | 2024-05-15T07:08:47.306Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
815 | 2024-05-15T07:08:47.306Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
816 | 2024-05-15T07:08:47.325Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
817 | 2024-05-15T07:08:47.325Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
818 | 2024-05-15T07:08:47.344Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
819 | 2024-05-15T07:08:47.344Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
820 | 2024-05-15T07:08:47.363Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
821 | 2024-05-15T07:08:47.363Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
822 | 2024-05-15T07:08:47.382Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
823 | 2024-05-15T07:08:47.382Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
824 | 2024-05-15T07:08:48.002Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
825 | 2024-05-15T07:08:48.002Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
826 | 2024-05-15T07:08:48.002Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
827 | 2024-05-15T07:08:48.002Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
828 | 2024-05-15T07:08:48.036Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
829 | 2024-05-15T07:08:48.036Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
830 | 2024-05-15T07:08:48.036Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
831 | 2024-05-15T07:08:48.036Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
832 | 2024-05-15T07:08:48.036Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
833 | 2024-05-15T07:08:48.036Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 184478
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 309cae2b-f25a-4047-bf88-9cd130250999
response_code = 500
uri = /omicron-zones
|
834 | 2024-05-15T07:08:48.036Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "309cae2b-f25a-4047-bf88-9cd130250999", "content-length": "124", "date": "Wed, 15 May 2024 07:08:48 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "309cae2b-f25a-4047-bf88-9cd130250999" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 317.193025ms
sled_address = [fd00:1122:3344:101::1]:12345
|
835 | 2024-05-15T07:08:48.353Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
836 | 2024-05-15T07:08:48.354Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
837 | 2024-05-15T07:08:48.354Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
838 | 2024-05-15T07:08:48.372Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
839 | 2024-05-15T07:08:48.372Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
840 | 2024-05-15T07:08:48.391Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
841 | 2024-05-15T07:08:48.391Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
842 | 2024-05-15T07:08:48.409Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
843 | 2024-05-15T07:08:48.409Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
844 | 2024-05-15T07:08:48.428Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
845 | 2024-05-15T07:08:48.428Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
846 | 2024-05-15T07:08:48.447Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
847 | 2024-05-15T07:08:48.447Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
848 | 2024-05-15T07:08:48.465Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
849 | 2024-05-15T07:08:48.465Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
850 | 2024-05-15T07:08:48.483Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
851 | 2024-05-15T07:08:48.483Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
852 | 2024-05-15T07:08:48.501Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
853 | 2024-05-15T07:08:48.501Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
854 | 2024-05-15T07:08:48.501Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
855 | 2024-05-15T07:08:48.502Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
856 | 2024-05-15T07:08:48.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
857 | 2024-05-15T07:08:48.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
858 | 2024-05-15T07:08:48.533Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
859 | 2024-05-15T07:08:48.534Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
860 | 2024-05-15T07:08:48.534Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
861 | 2024-05-15T07:08:48.534Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 180104
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = e765f94e-f27f-4e1f-9865-e5413cef99c2
response_code = 500
uri = /omicron-zones
|
862 | 2024-05-15T07:08:48.534Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e765f94e-f27f-4e1f-9865-e5413cef99c2", "content-length": "124", "date": "Wed, 15 May 2024 07:08:48 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e765f94e-f27f-4e1f-9865-e5413cef99c2" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 162.071826ms
sled_address = [fd00:1122:3344:101::1]:12345
|
863 | 2024-05-15T07:08:48.697Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
864 | 2024-05-15T07:08:48.698Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
865 | 2024-05-15T07:08:48.698Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
866 | 2024-05-15T07:08:48.716Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
867 | 2024-05-15T07:08:48.716Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
868 | 2024-05-15T07:08:48.734Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
869 | 2024-05-15T07:08:48.735Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
870 | 2024-05-15T07:08:48.753Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
871 | 2024-05-15T07:08:48.753Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
872 | 2024-05-15T07:08:48.770Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
873 | 2024-05-15T07:08:48.770Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
874 | 2024-05-15T07:08:48.788Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
875 | 2024-05-15T07:08:48.788Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
876 | 2024-05-15T07:08:48.805Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
877 | 2024-05-15T07:08:48.805Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
878 | 2024-05-15T07:08:48.823Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
879 | 2024-05-15T07:08:48.823Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
880 | 2024-05-15T07:08:48.840Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
881 | 2024-05-15T07:08:48.840Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
882 | 2024-05-15T07:08:48.841Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
883 | 2024-05-15T07:08:48.841Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
884 | 2024-05-15T07:08:48.873Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
885 | 2024-05-15T07:08:48.873Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
886 | 2024-05-15T07:08:48.873Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
887 | 2024-05-15T07:08:48.873Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
888 | 2024-05-15T07:08:48.873Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
889 | 2024-05-15T07:08:48.873Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 175780
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 2e2c06bc-a7c9-4461-b5d9-5eed21da466e
response_code = 500
uri = /omicron-zones
|
890 | 2024-05-15T07:08:48.873Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "2e2c06bc-a7c9-4461-b5d9-5eed21da466e", "content-length": "124", "date": "Wed, 15 May 2024 07:08:48 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "2e2c06bc-a7c9-4461-b5d9-5eed21da466e" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 496.32777ms
sled_address = [fd00:1122:3344:101::1]:12345
|
891 | 2024-05-15T07:08:49.371Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
892 | 2024-05-15T07:08:49.371Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
893 | 2024-05-15T07:08:49.371Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
894 | 2024-05-15T07:08:49.390Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
895 | 2024-05-15T07:08:49.390Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
896 | 2024-05-15T07:08:49.409Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
897 | 2024-05-15T07:08:49.409Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
898 | 2024-05-15T07:08:49.429Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
899 | 2024-05-15T07:08:49.429Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
900 | 2024-05-15T07:08:49.448Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
901 | 2024-05-15T07:08:49.448Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
902 | 2024-05-15T07:08:49.467Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
903 | 2024-05-15T07:08:49.467Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
904 | 2024-05-15T07:08:49.486Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
905 | 2024-05-15T07:08:49.486Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
906 | 2024-05-15T07:08:49.503Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
907 | 2024-05-15T07:08:49.503Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
908 | 2024-05-15T07:08:49.520Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
909 | 2024-05-15T07:08:49.520Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
910 | 2024-05-15T07:08:49.520Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
911 | 2024-05-15T07:08:49.520Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
912 | 2024-05-15T07:08:49.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
913 | 2024-05-15T07:08:49.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
914 | 2024-05-15T07:08:49.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
915 | 2024-05-15T07:08:49.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
916 | 2024-05-15T07:08:49.552Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
917 | 2024-05-15T07:08:49.552Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 180737
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = c4fe0d09-b3cf-4eb7-a452-510c355c4bdb
response_code = 500
uri = /omicron-zones
|
918 | 2024-05-15T07:08:49.552Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "c4fe0d09-b3cf-4eb7-a452-510c355c4bdb", "content-length": "124", "date": "Wed, 15 May 2024 07:08:49 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "c4fe0d09-b3cf-4eb7-a452-510c355c4bdb" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 583.687742ms
sled_address = [fd00:1122:3344:101::1]:12345
|
919 | 2024-05-15T07:08:50.136Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
920 | 2024-05-15T07:08:50.137Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
921 | 2024-05-15T07:08:50.137Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
922 | 2024-05-15T07:08:50.155Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
923 | 2024-05-15T07:08:50.155Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
924 | 2024-05-15T07:08:50.172Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
925 | 2024-05-15T07:08:50.172Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
926 | 2024-05-15T07:08:50.191Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
927 | 2024-05-15T07:08:50.191Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
928 | 2024-05-15T07:08:50.209Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
929 | 2024-05-15T07:08:50.209Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
930 | 2024-05-15T07:08:50.228Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
931 | 2024-05-15T07:08:50.228Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
932 | 2024-05-15T07:08:50.245Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
933 | 2024-05-15T07:08:50.245Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
934 | 2024-05-15T07:08:50.262Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
935 | 2024-05-15T07:08:50.262Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
936 | 2024-05-15T07:08:50.280Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
937 | 2024-05-15T07:08:50.280Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
938 | 2024-05-15T07:08:50.280Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
939 | 2024-05-15T07:08:50.280Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
940 | 2024-05-15T07:08:50.313Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
941 | 2024-05-15T07:08:50.313Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
942 | 2024-05-15T07:08:50.313Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
943 | 2024-05-15T07:08:50.313Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
944 | 2024-05-15T07:08:50.313Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
945 | 2024-05-15T07:08:50.313Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 176555
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = a0ef122c-7a62-43ed-88c5-a4c4a78a5811
response_code = 500
uri = /omicron-zones
|
946 | 2024-05-15T07:08:50.313Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a0ef122c-7a62-43ed-88c5-a4c4a78a5811", "content-length": "124", "date": "Wed, 15 May 2024 07:08:49 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a0ef122c-7a62-43ed-88c5-a4c4a78a5811" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 494.119885ms
sled_address = [fd00:1122:3344:101::1]:12345
|
947 | 2024-05-15T07:08:50.808Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
948 | 2024-05-15T07:08:50.809Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
949 | 2024-05-15T07:08:50.809Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
950 | 2024-05-15T07:08:50.828Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
951 | 2024-05-15T07:08:50.828Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
952 | 2024-05-15T07:08:50.846Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
953 | 2024-05-15T07:08:50.846Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
954 | 2024-05-15T07:08:50.866Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
955 | 2024-05-15T07:08:50.866Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
956 | 2024-05-15T07:08:50.883Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
957 | 2024-05-15T07:08:50.883Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
958 | 2024-05-15T07:08:50.902Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
959 | 2024-05-15T07:08:50.902Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
960 | 2024-05-15T07:08:50.919Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
961 | 2024-05-15T07:08:50.919Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
962 | 2024-05-15T07:08:50.937Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
963 | 2024-05-15T07:08:50.937Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
964 | 2024-05-15T07:08:50.955Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
965 | 2024-05-15T07:08:50.955Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
966 | 2024-05-15T07:08:50.956Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
967 | 2024-05-15T07:08:50.956Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
968 | 2024-05-15T07:08:50.988Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
969 | 2024-05-15T07:08:50.988Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
970 | 2024-05-15T07:08:50.988Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
971 | 2024-05-15T07:08:50.988Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
972 | 2024-05-15T07:08:50.988Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
973 | 2024-05-15T07:08:50.988Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 179716
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = bee4f337-00de-4ea4-bfa1-e2ba2a9dc547
response_code = 500
uri = /omicron-zones
|
974 | 2024-05-15T07:08:50.988Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "bee4f337-00de-4ea4-bfa1-e2ba2a9dc547", "content-length": "124", "date": "Wed, 15 May 2024 07:08:50 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "bee4f337-00de-4ea4-bfa1-e2ba2a9dc547" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 622.18083ms
sled_address = [fd00:1122:3344:101::1]:12345
|
975 | 2024-05-15T07:08:51.612Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
976 | 2024-05-15T07:08:51.612Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
977 | 2024-05-15T07:08:51.612Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
978 | 2024-05-15T07:08:51.633Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
979 | 2024-05-15T07:08:51.633Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
980 | 2024-05-15T07:08:51.653Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
981 | 2024-05-15T07:08:51.653Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
982 | 2024-05-15T07:08:51.672Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
983 | 2024-05-15T07:08:51.672Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
984 | 2024-05-15T07:08:51.694Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
985 | 2024-05-15T07:08:51.694Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
986 | 2024-05-15T07:08:51.714Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
987 | 2024-05-15T07:08:51.714Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
988 | 2024-05-15T07:08:51.732Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
989 | 2024-05-15T07:08:51.732Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
990 | 2024-05-15T07:08:51.751Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
991 | 2024-05-15T07:08:51.751Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
992 | 2024-05-15T07:08:51.771Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
993 | 2024-05-15T07:08:51.771Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
994 | 2024-05-15T07:08:51.771Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
995 | 2024-05-15T07:08:51.771Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
996 | 2024-05-15T07:08:51.805Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
997 | 2024-05-15T07:08:51.805Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
998 | 2024-05-15T07:08:51.805Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
999 | 2024-05-15T07:08:51.805Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1000 | 2024-05-15T07:08:51.805Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1001 | 2024-05-15T07:08:51.805Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 193369
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = f9479879-cc41-4b37-8658-dfcce586acd4
response_code = 500
uri = /omicron-zones
|
1002 | 2024-05-15T07:08:51.805Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f9479879-cc41-4b37-8658-dfcce586acd4", "content-length": "124", "date": "Wed, 15 May 2024 07:08:50 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f9479879-cc41-4b37-8658-dfcce586acd4" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 802.125606ms
sled_address = [fd00:1122:3344:101::1]:12345
|
1003 | 2024-05-15T07:08:52.609Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1004 | 2024-05-15T07:08:52.609Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1005 | 2024-05-15T07:08:52.609Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1006 | 2024-05-15T07:08:52.629Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1007 | 2024-05-15T07:08:52.629Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1008 | 2024-05-15T07:08:52.648Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1009 | 2024-05-15T07:08:52.648Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1010 | 2024-05-15T07:08:52.667Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1011 | 2024-05-15T07:08:52.667Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1012 | 2024-05-15T07:08:52.686Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1013 | 2024-05-15T07:08:52.686Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1014 | 2024-05-15T07:08:52.705Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1015 | 2024-05-15T07:08:52.705Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1016 | 2024-05-15T07:08:52.724Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1017 | 2024-05-15T07:08:52.724Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1018 | 2024-05-15T07:08:52.743Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1019 | 2024-05-15T07:08:52.743Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1020 | 2024-05-15T07:08:52.762Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1021 | 2024-05-15T07:08:52.762Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1022 | 2024-05-15T07:08:52.762Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1023 | 2024-05-15T07:08:52.762Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1024 | 2024-05-15T07:08:52.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1025 | 2024-05-15T07:08:52.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1026 | 2024-05-15T07:08:52.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1027 | 2024-05-15T07:08:52.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1028 | 2024-05-15T07:08:52.796Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1029 | 2024-05-15T07:08:52.796Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 186985
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 30c7abaf-6fbc-49a3-9280-e2d1ac8e5cf8
response_code = 500
uri = /omicron-zones
|
1030 | 2024-05-15T07:08:52.796Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "30c7abaf-6fbc-49a3-9280-e2d1ac8e5cf8", "content-length": "124", "date": "Wed, 15 May 2024 07:08:52 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "30c7abaf-6fbc-49a3-9280-e2d1ac8e5cf8" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 720.347015ms
sled_address = [fd00:1122:3344:101::1]:12345
|
1031 | 2024-05-15T07:08:53.518Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1032 | 2024-05-15T07:08:53.519Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1033 | 2024-05-15T07:08:53.519Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1034 | 2024-05-15T07:08:53.538Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1035 | 2024-05-15T07:08:53.538Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1036 | 2024-05-15T07:08:53.558Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1037 | 2024-05-15T07:08:53.558Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1038 | 2024-05-15T07:08:53.576Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1039 | 2024-05-15T07:08:53.576Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1040 | 2024-05-15T07:08:53.595Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1041 | 2024-05-15T07:08:53.595Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1042 | 2024-05-15T07:08:53.614Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1043 | 2024-05-15T07:08:53.614Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1044 | 2024-05-15T07:08:53.632Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1045 | 2024-05-15T07:08:53.632Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1046 | 2024-05-15T07:08:53.649Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1047 | 2024-05-15T07:08:53.649Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1048 | 2024-05-15T07:08:53.668Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1049 | 2024-05-15T07:08:53.668Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1050 | 2024-05-15T07:08:53.668Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1051 | 2024-05-15T07:08:53.668Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1052 | 2024-05-15T07:08:53.700Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1053 | 2024-05-15T07:08:53.700Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1054 | 2024-05-15T07:08:53.700Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1055 | 2024-05-15T07:08:53.700Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1056 | 2024-05-15T07:08:53.700Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1057 | 2024-05-15T07:08:53.700Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 182559
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = f810d3be-09e4-4edd-8da5-f9f4f8ad5a2d
response_code = 500
uri = /omicron-zones
|
1058 | 2024-05-15T07:08:53.701Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f810d3be-09e4-4edd-8da5-f9f4f8ad5a2d", "content-length": "124", "date": "Wed, 15 May 2024 07:08:53 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f810d3be-09e4-4edd-8da5-f9f4f8ad5a2d" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 981.080084ms
sled_address = [fd00:1122:3344:101::1]:12345
|
1059 | 2024-05-15T07:08:54.683Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1060 | 2024-05-15T07:08:54.684Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1061 | 2024-05-15T07:08:54.684Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1062 | 2024-05-15T07:08:54.704Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1063 | 2024-05-15T07:08:54.704Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1064 | 2024-05-15T07:08:54.724Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1065 | 2024-05-15T07:08:54.724Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1066 | 2024-05-15T07:08:54.743Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1067 | 2024-05-15T07:08:54.743Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1068 | 2024-05-15T07:08:54.763Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1069 | 2024-05-15T07:08:54.763Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1070 | 2024-05-15T07:08:54.782Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1071 | 2024-05-15T07:08:54.782Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1072 | 2024-05-15T07:08:54.801Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1073 | 2024-05-15T07:08:54.801Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1074 | 2024-05-15T07:08:54.820Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1075 | 2024-05-15T07:08:54.820Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1076 | 2024-05-15T07:08:54.839Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1077 | 2024-05-15T07:08:54.839Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1078 | 2024-05-15T07:08:54.839Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1079 | 2024-05-15T07:08:54.839Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1080 | 2024-05-15T07:08:54.873Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1081 | 2024-05-15T07:08:54.873Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1082 | 2024-05-15T07:08:54.873Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1083 | 2024-05-15T07:08:54.873Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1084 | 2024-05-15T07:08:54.873Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1085 | 2024-05-15T07:08:54.873Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 189556
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = be5c5b64-aaf1-4b3a-a770-0cd86853563f
response_code = 500
uri = /omicron-zones
|
1086 | 2024-05-15T07:08:54.873Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "be5c5b64-aaf1-4b3a-a770-0cd86853563f", "content-length": "124", "date": "Wed, 15 May 2024 07:08:54 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "be5c5b64-aaf1-4b3a-a770-0cd86853563f" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 1.679257772s
sled_address = [fd00:1122:3344:101::1]:12345
|
1087 | 2024-05-15T07:08:56.341Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 166.451793ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:426
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
1088 | 2024-05-15T07:08:56.554Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1089 | 2024-05-15T07:08:56.554Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1090 | 2024-05-15T07:08:56.554Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1091 | 2024-05-15T07:08:56.576Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1092 | 2024-05-15T07:08:56.576Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1093 | 2024-05-15T07:08:56.595Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1094 | 2024-05-15T07:08:56.595Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1095 | 2024-05-15T07:08:56.614Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1096 | 2024-05-15T07:08:56.614Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1097 | 2024-05-15T07:08:56.633Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1098 | 2024-05-15T07:08:56.633Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1099 | 2024-05-15T07:08:56.652Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1100 | 2024-05-15T07:08:56.652Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1101 | 2024-05-15T07:08:56.671Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1102 | 2024-05-15T07:08:56.671Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1103 | 2024-05-15T07:08:56.689Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1104 | 2024-05-15T07:08:56.689Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1105 | 2024-05-15T07:08:56.708Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1106 | 2024-05-15T07:08:56.708Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1107 | 2024-05-15T07:08:56.709Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1108 | 2024-05-15T07:08:56.709Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1109 | 2024-05-15T07:08:56.743Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1110 | 2024-05-15T07:08:56.743Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1111 | 2024-05-15T07:08:56.743Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1112 | 2024-05-15T07:08:56.743Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1113 | 2024-05-15T07:08:56.743Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1114 | 2024-05-15T07:08:56.743Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 188960
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 6d962ca5-9034-422f-a2b2-e036abeb0d71
response_code = 500
uri = /omicron-zones
|
1115 | 2024-05-15T07:08:56.743Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6d962ca5-9034-422f-a2b2-e036abeb0d71", "content-length": "124", "date": "Wed, 15 May 2024 07:08:56 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6d962ca5-9034-422f-a2b2-e036abeb0d71" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 1.228987941s
sled_address = [fd00:1122:3344:101::1]:12345
|
1116 | 2024-05-15T07:08:57.973Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1117 | 2024-05-15T07:08:57.974Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1118 | 2024-05-15T07:08:57.974Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1119 | 2024-05-15T07:08:57.995Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1120 | 2024-05-15T07:08:57.995Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1121 | 2024-05-15T07:08:58.016Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1122 | 2024-05-15T07:08:58.016Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1123 | 2024-05-15T07:08:58.035Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1124 | 2024-05-15T07:08:58.035Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1125 | 2024-05-15T07:08:58.055Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1126 | 2024-05-15T07:08:58.055Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1127 | 2024-05-15T07:08:58.074Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1128 | 2024-05-15T07:08:58.074Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1129 | 2024-05-15T07:08:58.093Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1130 | 2024-05-15T07:08:58.093Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1131 | 2024-05-15T07:08:58.113Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1132 | 2024-05-15T07:08:58.113Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1133 | 2024-05-15T07:08:58.132Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1134 | 2024-05-15T07:08:58.132Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1135 | 2024-05-15T07:08:58.132Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1136 | 2024-05-15T07:08:58.132Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1137 | 2024-05-15T07:08:58.166Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1138 | 2024-05-15T07:08:58.166Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1139 | 2024-05-15T07:08:58.166Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1140 | 2024-05-15T07:08:58.166Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1141 | 2024-05-15T07:08:58.166Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1142 | 2024-05-15T07:08:58.166Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 193379
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = c19a2e72-69e2-4673-82f6-299687639c71
response_code = 500
uri = /omicron-zones
|
1143 | 2024-05-15T07:08:58.166Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "c19a2e72-69e2-4673-82f6-299687639c71", "content-length": "124", "date": "Wed, 15 May 2024 07:08:58 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "c19a2e72-69e2-4673-82f6-299687639c71" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 2.594466711s
sled_address = [fd00:1122:3344:101::1]:12345
|
1144 | 2024-05-15T07:09:00.762Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1145 | 2024-05-15T07:09:00.763Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1146 | 2024-05-15T07:09:00.763Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1147 | 2024-05-15T07:09:00.783Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1148 | 2024-05-15T07:09:00.783Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1149 | 2024-05-15T07:09:00.803Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1150 | 2024-05-15T07:09:00.803Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1151 | 2024-05-15T07:09:00.821Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1152 | 2024-05-15T07:09:00.821Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1153 | 2024-05-15T07:09:00.842Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1154 | 2024-05-15T07:09:00.842Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1155 | 2024-05-15T07:09:00.860Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1156 | 2024-05-15T07:09:00.860Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1157 | 2024-05-15T07:09:00.879Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1158 | 2024-05-15T07:09:00.879Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1159 | 2024-05-15T07:09:00.898Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1160 | 2024-05-15T07:09:00.898Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1161 | 2024-05-15T07:09:00.917Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1162 | 2024-05-15T07:09:00.917Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1163 | 2024-05-15T07:09:00.917Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1164 | 2024-05-15T07:09:00.917Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1165 | 2024-05-15T07:09:00.950Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1166 | 2024-05-15T07:09:00.950Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1167 | 2024-05-15T07:09:00.950Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1168 | 2024-05-15T07:09:00.950Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1169 | 2024-05-15T07:09:00.950Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1170 | 2024-05-15T07:09:00.950Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 187276
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 11266bbc-2630-4db7-adf9-bced0d5687ee
response_code = 500
uri = /omicron-zones
|
1171 | 2024-05-15T07:09:00.950Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "11266bbc-2630-4db7-adf9-bced0d5687ee", "content-length": "124", "date": "Wed, 15 May 2024 07:09:00 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "11266bbc-2630-4db7-adf9-bced0d5687ee" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 2.481282364s
sled_address = [fd00:1122:3344:101::1]:12345
|
1172 | 2024-05-15T07:09:03.209Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:47579
|
1173 | 2024-05-15T07:09:03.209Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 72
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:47579
req_id = d0393f44-c042-4cef-93a6-09985ad997e4
response_code = 200
uri = /baseboard
|
1174 | 2024-05-15T07:09:03.432Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1175 | 2024-05-15T07:09:03.432Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1176 | 2024-05-15T07:09:03.432Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1177 | 2024-05-15T07:09:03.453Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1178 | 2024-05-15T07:09:03.453Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1179 | 2024-05-15T07:09:03.473Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1180 | 2024-05-15T07:09:03.473Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1181 | 2024-05-15T07:09:03.493Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1182 | 2024-05-15T07:09:03.493Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1183 | 2024-05-15T07:09:03.513Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1184 | 2024-05-15T07:09:03.513Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1185 | 2024-05-15T07:09:03.533Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1186 | 2024-05-15T07:09:03.533Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1187 | 2024-05-15T07:09:03.551Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1188 | 2024-05-15T07:09:03.551Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1189 | 2024-05-15T07:09:03.571Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1190 | 2024-05-15T07:09:03.571Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1191 | 2024-05-15T07:09:03.590Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1192 | 2024-05-15T07:09:03.590Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1193 | 2024-05-15T07:09:03.590Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1194 | 2024-05-15T07:09:03.590Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1195 | 2024-05-15T07:09:03.624Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1196 | 2024-05-15T07:09:03.624Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1197 | 2024-05-15T07:09:03.624Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1198 | 2024-05-15T07:09:03.624Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1199 | 2024-05-15T07:09:03.624Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1200 | 2024-05-15T07:09:03.624Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 192497
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 6632bf65-1d4d-4339-a9cc-462c12182c40
response_code = 500
uri = /omicron-zones
|
1201 | 2024-05-15T07:09:03.625Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6632bf65-1d4d-4339-a9cc-462c12182c40", "content-length": "124", "date": "Wed, 15 May 2024 07:09:03 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6632bf65-1d4d-4339-a9cc-462c12182c40" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 3.096175391s
sled_address = [fd00:1122:3344:101::1]:12345
|
1202 | 2024-05-15T07:09:06.722Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1203 | 2024-05-15T07:09:06.722Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1204 | 2024-05-15T07:09:06.722Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1205 | 2024-05-15T07:09:06.745Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1206 | 2024-05-15T07:09:06.745Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1207 | 2024-05-15T07:09:06.765Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1208 | 2024-05-15T07:09:06.765Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1209 | 2024-05-15T07:09:06.784Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1210 | 2024-05-15T07:09:06.784Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1211 | 2024-05-15T07:09:06.804Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1212 | 2024-05-15T07:09:06.804Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1213 | 2024-05-15T07:09:06.823Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1214 | 2024-05-15T07:09:06.823Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1215 | 2024-05-15T07:09:06.841Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1216 | 2024-05-15T07:09:06.841Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1217 | 2024-05-15T07:09:06.861Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1218 | 2024-05-15T07:09:06.861Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1219 | 2024-05-15T07:09:06.880Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1220 | 2024-05-15T07:09:06.880Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1221 | 2024-05-15T07:09:06.880Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1222 | 2024-05-15T07:09:06.881Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1223 | 2024-05-15T07:09:06.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1224 | 2024-05-15T07:09:06.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1225 | 2024-05-15T07:09:06.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1226 | 2024-05-15T07:09:06.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1227 | 2024-05-15T07:09:06.916Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1228 | 2024-05-15T07:09:06.916Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 194219
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 50753cb7-ee78-4b91-8bce-132c0612f5a8
response_code = 500
uri = /omicron-zones
|
1229 | 2024-05-15T07:09:06.917Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "50753cb7-ee78-4b91-8bce-132c0612f5a8", "content-length": "124", "date": "Wed, 15 May 2024 07:09:06 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "50753cb7-ee78-4b91-8bce-132c0612f5a8" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 3.294517183s
sled_address = [fd00:1122:3344:101::1]:12345
|
1230 | 2024-05-15T07:09:10.212Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1231 | 2024-05-15T07:09:10.212Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1232 | 2024-05-15T07:09:10.212Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1233 | 2024-05-15T07:09:10.233Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1234 | 2024-05-15T07:09:10.233Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1235 | 2024-05-15T07:09:10.254Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1236 | 2024-05-15T07:09:10.254Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1237 | 2024-05-15T07:09:10.272Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1238 | 2024-05-15T07:09:10.272Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1239 | 2024-05-15T07:09:10.290Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1240 | 2024-05-15T07:09:10.290Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1241 | 2024-05-15T07:09:10.310Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1242 | 2024-05-15T07:09:10.310Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1243 | 2024-05-15T07:09:10.328Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1244 | 2024-05-15T07:09:10.328Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1245 | 2024-05-15T07:09:10.347Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1246 | 2024-05-15T07:09:10.347Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1247 | 2024-05-15T07:09:10.364Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1248 | 2024-05-15T07:09:10.364Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1249 | 2024-05-15T07:09:10.365Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1250 | 2024-05-15T07:09:10.365Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1251 | 2024-05-15T07:09:10.397Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1252 | 2024-05-15T07:09:10.397Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1253 | 2024-05-15T07:09:10.397Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1254 | 2024-05-15T07:09:10.397Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1255 | 2024-05-15T07:09:10.397Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1256 | 2024-05-15T07:09:10.397Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 185466
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = badd8074-ffbb-4e77-9958-593538c0a44f
response_code = 500
uri = /omicron-zones
|
1257 | 2024-05-15T07:09:10.398Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "badd8074-ffbb-4e77-9958-593538c0a44f", "content-length": "124", "date": "Wed, 15 May 2024 07:09:10 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "badd8074-ffbb-4e77-9958-593538c0a44f" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 3.974484717s
sled_address = [fd00:1122:3344:101::1]:12345
|
1258 | 2024-05-15T07:09:11.542Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 654.623942ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:426
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
1259 | 2024-05-15T07:09:14.373Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1260 | 2024-05-15T07:09:14.374Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1261 | 2024-05-15T07:09:14.374Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1262 | 2024-05-15T07:09:14.396Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1263 | 2024-05-15T07:09:14.396Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1264 | 2024-05-15T07:09:14.416Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1265 | 2024-05-15T07:09:14.416Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1266 | 2024-05-15T07:09:14.435Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1267 | 2024-05-15T07:09:14.435Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1268 | 2024-05-15T07:09:14.454Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1269 | 2024-05-15T07:09:14.455Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1270 | 2024-05-15T07:09:14.474Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1271 | 2024-05-15T07:09:14.474Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1272 | 2024-05-15T07:09:14.493Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1273 | 2024-05-15T07:09:14.493Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1274 | 2024-05-15T07:09:14.512Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1275 | 2024-05-15T07:09:14.512Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1276 | 2024-05-15T07:09:14.531Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1277 | 2024-05-15T07:09:14.531Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1278 | 2024-05-15T07:09:14.531Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1279 | 2024-05-15T07:09:14.531Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1280 | 2024-05-15T07:09:14.564Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1281 | 2024-05-15T07:09:14.564Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1282 | 2024-05-15T07:09:14.564Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1283 | 2024-05-15T07:09:14.564Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1284 | 2024-05-15T07:09:14.565Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1285 | 2024-05-15T07:09:14.565Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 191798
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 590619cc-fa01-4d79-91f3-43e20261f66b
response_code = 500
uri = /omicron-zones
|
1286 | 2024-05-15T07:09:14.565Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "590619cc-fa01-4d79-91f3-43e20261f66b", "content-length": "124", "date": "Wed, 15 May 2024 07:09:14 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "590619cc-fa01-4d79-91f3-43e20261f66b" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 5.888114464s
sled_address = [fd00:1122:3344:101::1]:12345
|
1287 | 2024-05-15T07:09:20.454Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1288 | 2024-05-15T07:09:20.455Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1289 | 2024-05-15T07:09:20.455Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1290 | 2024-05-15T07:09:20.477Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1291 | 2024-05-15T07:09:20.477Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1292 | 2024-05-15T07:09:20.497Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1293 | 2024-05-15T07:09:20.497Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1294 | 2024-05-15T07:09:20.516Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1295 | 2024-05-15T07:09:20.516Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1296 | 2024-05-15T07:09:20.535Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1297 | 2024-05-15T07:09:20.535Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1298 | 2024-05-15T07:09:20.553Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1299 | 2024-05-15T07:09:20.553Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1300 | 2024-05-15T07:09:20.572Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1301 | 2024-05-15T07:09:20.572Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1302 | 2024-05-15T07:09:20.590Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1303 | 2024-05-15T07:09:20.590Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1304 | 2024-05-15T07:09:20.608Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1305 | 2024-05-15T07:09:20.608Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1306 | 2024-05-15T07:09:20.608Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1307 | 2024-05-15T07:09:20.609Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1308 | 2024-05-15T07:09:20.641Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1309 | 2024-05-15T07:09:20.641Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1310 | 2024-05-15T07:09:20.641Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1311 | 2024-05-15T07:09:20.641Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1312 | 2024-05-15T07:09:20.641Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1313 | 2024-05-15T07:09:20.641Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 186881
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = dc21f78c-1282-451b-89d5-5c288e9ab8e1
response_code = 500
uri = /omicron-zones
|
1314 | 2024-05-15T07:09:20.641Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "dc21f78c-1282-451b-89d5-5c288e9ab8e1", "content-length": "124", "date": "Wed, 15 May 2024 07:09:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "dc21f78c-1282-451b-89d5-5c288e9ab8e1" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 4.32672259s
sled_address = [fd00:1122:3344:101::1]:12345
|
1315 | 2024-05-15T07:09:24.969Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1316 | 2024-05-15T07:09:24.969Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1317 | 2024-05-15T07:09:24.969Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1318 | 2024-05-15T07:09:24.991Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1319 | 2024-05-15T07:09:24.991Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1320 | 2024-05-15T07:09:25.011Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1321 | 2024-05-15T07:09:25.011Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1322 | 2024-05-15T07:09:25.030Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1323 | 2024-05-15T07:09:25.030Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1324 | 2024-05-15T07:09:25.048Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1325 | 2024-05-15T07:09:25.048Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1326 | 2024-05-15T07:09:25.068Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1327 | 2024-05-15T07:09:25.068Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1328 | 2024-05-15T07:09:25.086Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1329 | 2024-05-15T07:09:25.086Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1330 | 2024-05-15T07:09:25.106Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1331 | 2024-05-15T07:09:25.106Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1332 | 2024-05-15T07:09:25.124Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1333 | 2024-05-15T07:09:25.124Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1334 | 2024-05-15T07:09:25.125Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1335 | 2024-05-15T07:09:25.125Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1336 | 2024-05-15T07:09:25.159Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1337 | 2024-05-15T07:09:25.159Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1338 | 2024-05-15T07:09:25.159Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1339 | 2024-05-15T07:09:25.159Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1340 | 2024-05-15T07:09:25.159Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1341 | 2024-05-15T07:09:25.159Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 190140
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = a7738156-be03-48c6-a58c-b1cf055c4c03
response_code = 500
uri = /omicron-zones
|
1342 | 2024-05-15T07:09:25.159Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a7738156-be03-48c6-a58c-b1cf055c4c03", "content-length": "124", "date": "Wed, 15 May 2024 07:09:24 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a7738156-be03-48c6-a58c-b1cf055c4c03" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 7.9695987s
sled_address = [fd00:1122:3344:101::1]:12345
|
1343 | 2024-05-15T07:09:27.230Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 648.441642ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:426
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
1344 | 2024-05-15T07:09:33.129Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1345 | 2024-05-15T07:09:33.130Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1346 | 2024-05-15T07:09:33.130Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1347 | 2024-05-15T07:09:33.153Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1348 | 2024-05-15T07:09:33.153Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1349 | 2024-05-15T07:09:33.174Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1350 | 2024-05-15T07:09:33.174Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1351 | 2024-05-15T07:09:33.193Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1352 | 2024-05-15T07:09:33.193Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1353 | 2024-05-15T07:09:33.213Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1354 | 2024-05-15T07:09:33.213Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1355 | 2024-05-15T07:09:33.233Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1356 | 2024-05-15T07:09:33.233Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1357 | 2024-05-15T07:09:33.242Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:35017
|
1358 | 2024-05-15T07:09:33.242Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 101
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:35017
req_id = e17b0547-f814-4e8e-8ace-a5e1294af997
response_code = 200
uri = /baseboard
|
1359 | 2024-05-15T07:09:33.252Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1360 | 2024-05-15T07:09:33.253Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1361 | 2024-05-15T07:09:33.272Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1362 | 2024-05-15T07:09:33.272Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1363 | 2024-05-15T07:09:33.290Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1364 | 2024-05-15T07:09:33.291Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1365 | 2024-05-15T07:09:33.291Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1366 | 2024-05-15T07:09:33.291Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1367 | 2024-05-15T07:09:33.324Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1368 | 2024-05-15T07:09:33.324Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1369 | 2024-05-15T07:09:33.324Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1370 | 2024-05-15T07:09:33.324Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1371 | 2024-05-15T07:09:33.324Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1372 | 2024-05-15T07:09:33.325Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 194942
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = f3b51397-f65b-48e2-82fd-3553e1e78d2f
response_code = 500
uri = /omicron-zones
|
1373 | 2024-05-15T07:09:33.325Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f3b51397-f65b-48e2-82fd-3553e1e78d2f", "content-length": "124", "date": "Wed, 15 May 2024 07:09:33 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f3b51397-f65b-48e2-82fd-3553e1e78d2f" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 6.279875667s
sled_address = [fd00:1122:3344:101::1]:12345
|
1374 | 2024-05-15T07:09:39.606Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1375 | 2024-05-15T07:09:39.607Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1376 | 2024-05-15T07:09:39.608Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1377 | 2024-05-15T07:09:39.630Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1378 | 2024-05-15T07:09:39.630Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1379 | 2024-05-15T07:09:39.650Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1380 | 2024-05-15T07:09:39.650Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1381 | 2024-05-15T07:09:39.669Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1382 | 2024-05-15T07:09:39.669Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1383 | 2024-05-15T07:09:39.689Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1384 | 2024-05-15T07:09:39.689Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1385 | 2024-05-15T07:09:39.708Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1386 | 2024-05-15T07:09:39.708Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1387 | 2024-05-15T07:09:39.727Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1388 | 2024-05-15T07:09:39.727Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1389 | 2024-05-15T07:09:39.746Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1390 | 2024-05-15T07:09:39.746Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1391 | 2024-05-15T07:09:39.764Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1392 | 2024-05-15T07:09:39.765Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1393 | 2024-05-15T07:09:39.765Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1394 | 2024-05-15T07:09:39.765Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1395 | 2024-05-15T07:09:39.799Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1396 | 2024-05-15T07:09:39.799Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1397 | 2024-05-15T07:09:39.799Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1398 | 2024-05-15T07:09:39.799Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1399 | 2024-05-15T07:09:39.799Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1400 | 2024-05-15T07:09:39.799Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 192427
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = bd23c28d-fcea-4ce3-a07e-a3d37f7bb73d
response_code = 500
uri = /omicron-zones
|
1401 | 2024-05-15T07:09:39.799Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "bd23c28d-fcea-4ce3-a07e-a3d37f7bb73d", "content-length": "124", "date": "Wed, 15 May 2024 07:09:39 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "bd23c28d-fcea-4ce3-a07e-a3d37f7bb73d" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 12.93951738s
sled_address = [fd00:1122:3344:101::1]:12345
|
1402 | 2024-05-15T07:09:42.913Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 1.036520077s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:426
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
1403 | 2024-05-15T07:09:52.803Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1404 | 2024-05-15T07:09:52.803Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1405 | 2024-05-15T07:09:52.803Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1406 | 2024-05-15T07:09:52.826Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1407 | 2024-05-15T07:09:52.826Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1408 | 2024-05-15T07:09:52.845Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1409 | 2024-05-15T07:09:52.845Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1410 | 2024-05-15T07:09:52.864Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1411 | 2024-05-15T07:09:52.864Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1412 | 2024-05-15T07:09:52.881Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1413 | 2024-05-15T07:09:52.882Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1414 | 2024-05-15T07:09:52.900Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1415 | 2024-05-15T07:09:52.900Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1416 | 2024-05-15T07:09:52.918Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1417 | 2024-05-15T07:09:52.918Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1418 | 2024-05-15T07:09:52.936Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1419 | 2024-05-15T07:09:52.936Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1420 | 2024-05-15T07:09:52.954Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1421 | 2024-05-15T07:09:52.954Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1422 | 2024-05-15T07:09:52.954Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1423 | 2024-05-15T07:09:52.954Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1424 | 2024-05-15T07:09:52.988Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1425 | 2024-05-15T07:09:52.988Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1426 | 2024-05-15T07:09:52.988Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1427 | 2024-05-15T07:09:52.988Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1428 | 2024-05-15T07:09:52.988Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1429 | 2024-05-15T07:09:52.988Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 185022
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 52227679-a170-4104-9267-bb76fbea8e80
response_code = 500
uri = /omicron-zones
|
1430 | 2024-05-15T07:09:52.988Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "52227679-a170-4104-9267-bb76fbea8e80", "content-length": "124", "date": "Wed, 15 May 2024 07:09:52 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "52227679-a170-4104-9267-bb76fbea8e80" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 8.985632361s
sled_address = [fd00:1122:3344:101::1]:12345
|
1431 | 2024-05-15T07:09:59.031Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 5.907476941s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:426
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
1432 | 2024-05-15T07:10:01.951Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1433 | 2024-05-15T07:10:01.951Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1434 | 2024-05-15T07:10:01.951Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1435 | 2024-05-15T07:10:01.972Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1436 | 2024-05-15T07:10:01.972Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1437 | 2024-05-15T07:10:01.992Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1438 | 2024-05-15T07:10:01.992Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1439 | 2024-05-15T07:10:02.012Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1440 | 2024-05-15T07:10:02.012Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1441 | 2024-05-15T07:10:02.030Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1442 | 2024-05-15T07:10:02.030Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1443 | 2024-05-15T07:10:02.050Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1444 | 2024-05-15T07:10:02.050Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1445 | 2024-05-15T07:10:02.068Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1446 | 2024-05-15T07:10:02.068Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1447 | 2024-05-15T07:10:02.087Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1448 | 2024-05-15T07:10:02.087Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1449 | 2024-05-15T07:10:02.105Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1450 | 2024-05-15T07:10:02.105Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1451 | 2024-05-15T07:10:02.105Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1452 | 2024-05-15T07:10:02.105Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1453 | 2024-05-15T07:10:02.137Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1454 | 2024-05-15T07:10:02.137Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1455 | 2024-05-15T07:10:02.137Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1456 | 2024-05-15T07:10:02.137Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1457 | 2024-05-15T07:10:02.137Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1458 | 2024-05-15T07:10:02.137Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 186515
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = d1a65eb4-8eab-4f08-9485-288e93121900
response_code = 500
uri = /omicron-zones
|
1459 | 2024-05-15T07:10:02.137Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d1a65eb4-8eab-4f08-9485-288e93121900", "content-length": "124", "date": "Wed, 15 May 2024 07:10:01 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d1a65eb4-8eab-4f08-9485-288e93121900" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 14.726985305s
sled_address = [fd00:1122:3344:101::1]:12345
|
1460 | 2024-05-15T07:10:03.311Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:48092
|
1461 | 2024-05-15T07:10:03.312Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 62
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:48092
req_id = d532c044-f0b2-4dc7-ada1-548a4203a694
response_code = 200
uri = /baseboard
|
1462 | 2024-05-15T07:10:16.825Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1463 | 2024-05-15T07:10:16.826Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1464 | 2024-05-15T07:10:16.826Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1465 | 2024-05-15T07:10:16.847Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1466 | 2024-05-15T07:10:16.847Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1467 | 2024-05-15T07:10:16.866Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1468 | 2024-05-15T07:10:16.866Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1469 | 2024-05-15T07:10:16.884Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1470 | 2024-05-15T07:10:16.884Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1471 | 2024-05-15T07:10:16.903Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1472 | 2024-05-15T07:10:16.903Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1473 | 2024-05-15T07:10:16.923Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1474 | 2024-05-15T07:10:16.923Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1475 | 2024-05-15T07:10:16.943Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1476 | 2024-05-15T07:10:16.943Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1477 | 2024-05-15T07:10:16.962Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1478 | 2024-05-15T07:10:16.962Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1479 | 2024-05-15T07:10:16.980Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1480 | 2024-05-15T07:10:16.980Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1481 | 2024-05-15T07:10:16.980Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1482 | 2024-05-15T07:10:16.980Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1483 | 2024-05-15T07:10:17.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1484 | 2024-05-15T07:10:17.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1485 | 2024-05-15T07:10:17.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1486 | 2024-05-15T07:10:17.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1487 | 2024-05-15T07:10:17.012Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1488 | 2024-05-15T07:10:17.012Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 187402
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 222904d9-f1cd-4938-842c-ae8b29a1b6e3
response_code = 500
uri = /omicron-zones
|
1489 | 2024-05-15T07:10:17.013Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "222904d9-f1cd-4938-842c-ae8b29a1b6e3", "content-length": "124", "date": "Wed, 15 May 2024 07:10:17 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "222904d9-f1cd-4938-842c-ae8b29a1b6e3" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 8.884581932s
sled_address = [fd00:1122:3344:101::1]:12345
|
1490 | 2024-05-15T07:10:19.920Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 6.250521478s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:426
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
1491 | 2024-05-15T07:10:25.874Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1492 | 2024-05-15T07:10:25.874Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1493 | 2024-05-15T07:10:25.874Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1494 | 2024-05-15T07:10:25.897Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1495 | 2024-05-15T07:10:25.897Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1496 | 2024-05-15T07:10:25.918Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1497 | 2024-05-15T07:10:25.918Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1498 | 2024-05-15T07:10:25.937Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1499 | 2024-05-15T07:10:25.937Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1500 | 2024-05-15T07:10:25.957Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1501 | 2024-05-15T07:10:25.957Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1502 | 2024-05-15T07:10:25.975Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1503 | 2024-05-15T07:10:25.976Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1504 | 2024-05-15T07:10:25.994Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1505 | 2024-05-15T07:10:25.994Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1506 | 2024-05-15T07:10:26.012Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1507 | 2024-05-15T07:10:26.012Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1508 | 2024-05-15T07:10:26.031Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1509 | 2024-05-15T07:10:26.031Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1510 | 2024-05-15T07:10:26.031Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1511 | 2024-05-15T07:10:26.031Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1512 | 2024-05-15T07:10:26.064Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1513 | 2024-05-15T07:10:26.064Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1514 | 2024-05-15T07:10:26.064Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1515 | 2024-05-15T07:10:26.064Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1516 | 2024-05-15T07:10:26.064Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1517 | 2024-05-15T07:10:26.064Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 190529
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 9e6652a9-7858-47c6-a022-ed7c9ce2ffd3
response_code = 500
uri = /omicron-zones
|
1518 | 2024-05-15T07:10:26.064Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "9e6652a9-7858-47c6-a022-ed7c9ce2ffd3", "content-length": "124", "date": "Wed, 15 May 2024 07:10:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "9e6652a9-7858-47c6-a022-ed7c9ce2ffd3" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 11.853351532s
sled_address = [fd00:1122:3344:101::1]:12345
|
1519 | 2024-05-15T07:10:33.262Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45586
|
1520 | 2024-05-15T07:10:33.262Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 46
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45586
req_id = 7141a425-6bde-4eff-b697-c1954bf104dc
response_code = 200
uri = /baseboard
|
1521 | 2024-05-15T07:10:37.886Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1522 | 2024-05-15T07:10:37.887Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1523 | 2024-05-15T07:10:37.887Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1524 | 2024-05-15T07:10:37.909Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1525 | 2024-05-15T07:10:37.909Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1526 | 2024-05-15T07:10:37.928Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1527 | 2024-05-15T07:10:37.928Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1528 | 2024-05-15T07:10:37.948Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1529 | 2024-05-15T07:10:37.948Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1530 | 2024-05-15T07:10:37.967Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1531 | 2024-05-15T07:10:37.968Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1532 | 2024-05-15T07:10:37.987Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1533 | 2024-05-15T07:10:37.987Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1534 | 2024-05-15T07:10:38.008Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1535 | 2024-05-15T07:10:38.008Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1536 | 2024-05-15T07:10:38.027Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1537 | 2024-05-15T07:10:38.027Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1538 | 2024-05-15T07:10:38.046Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1539 | 2024-05-15T07:10:38.046Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1540 | 2024-05-15T07:10:38.046Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1541 | 2024-05-15T07:10:38.046Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1542 | 2024-05-15T07:10:38.079Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1543 | 2024-05-15T07:10:38.079Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1544 | 2024-05-15T07:10:38.079Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1545 | 2024-05-15T07:10:38.079Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1546 | 2024-05-15T07:10:38.079Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1547 | 2024-05-15T07:10:38.079Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 193536
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 87e360d0-8300-42ab-9b80-00ad1d96f466
response_code = 500
uri = /omicron-zones
|
1548 | 2024-05-15T07:10:38.080Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "87e360d0-8300-42ab-9b80-00ad1d96f466", "content-length": "124", "date": "Wed, 15 May 2024 07:10:38 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "87e360d0-8300-42ab-9b80-00ad1d96f466" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 13.888696225s
sled_address = [fd00:1122:3344:101::1]:12345
|
1549 | 2024-05-15T07:10:41.151Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 23.476752366s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:426
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
1550 | 2024-05-15T07:10:51.932Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1551 | 2024-05-15T07:10:51.933Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1552 | 2024-05-15T07:10:51.933Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1553 | 2024-05-15T07:10:51.954Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1554 | 2024-05-15T07:10:51.954Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1555 | 2024-05-15T07:10:51.972Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1556 | 2024-05-15T07:10:51.972Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1557 | 2024-05-15T07:10:51.990Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1558 | 2024-05-15T07:10:51.990Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1559 | 2024-05-15T07:10:52.007Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1560 | 2024-05-15T07:10:52.007Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1561 | 2024-05-15T07:10:52.025Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1562 | 2024-05-15T07:10:52.025Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1563 | 2024-05-15T07:10:52.042Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1564 | 2024-05-15T07:10:52.042Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1565 | 2024-05-15T07:10:52.060Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1566 | 2024-05-15T07:10:52.060Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1567 | 2024-05-15T07:10:52.077Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1568 | 2024-05-15T07:10:52.077Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1569 | 2024-05-15T07:10:52.077Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1570 | 2024-05-15T07:10:52.078Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1571 | 2024-05-15T07:10:52.109Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1572 | 2024-05-15T07:10:52.109Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1573 | 2024-05-15T07:10:52.109Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1574 | 2024-05-15T07:10:52.109Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1575 | 2024-05-15T07:10:52.109Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3030
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1576 | 2024-05-15T07:10:52.109Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015", TimeNotSynchronized), ("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14", TimeNotSynchronized), ("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3", TimeNotSynchronized), ("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9", TimeNotSynchronized), ("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866
latency_us = 177019
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 4af6b0ca-c76e-4ca5-9cda-41b92be9948c
response_code = 500
uri = /omicron-zones
|
1577 | 2024-05-15T07:10:52.109Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "4af6b0ca-c76e-4ca5-9cda-41b92be9948c", "content-length": "124", "date": "Wed, 15 May 2024 07:10:51 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "4af6b0ca-c76e-4ca5-9cda-41b92be9948c" }
file = sled-agent/src/rack_setup/service.rs:468
retry_after = 29.427935156s
sled_address = [fd00:1122:3344:101::1]:12345
|
1578 | 2024-05-15T07:11:03.216Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:46512
|
1579 | 2024-05-15T07:11:03.216Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 73
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:46512
req_id = a1ede15f-1eae-4837-b47b-ca4642768282
response_code = 200
uri = /baseboard
|
1580 | 2024-05-15T07:11:19.559Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 29.914242547s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:426
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
1581 | 2024-05-15T07:11:21.459Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1582 | 2024-05-15T07:11:21.459Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1583 | 2024-05-15T07:11:21.460Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1584 | 2024-05-15T07:11:21.482Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1585 | 2024-05-15T07:11:21.482Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1586 | 2024-05-15T07:11:21.503Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1587 | 2024-05-15T07:11:21.503Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1588 | 2024-05-15T07:11:21.522Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1589 | 2024-05-15T07:11:21.522Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1590 | 2024-05-15T07:11:21.541Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1591 | 2024-05-15T07:11:21.541Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1592 | 2024-05-15T07:11:21.560Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1593 | 2024-05-15T07:11:21.560Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1594 | 2024-05-15T07:11:21.579Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1595 | 2024-05-15T07:11:21.579Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1596 | 2024-05-15T07:11:21.597Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1597 | 2024-05-15T07:11:21.597Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1598 | 2024-05-15T07:11:21.614Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1599 | 2024-05-15T07:11:21.614Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1600 | 2024-05-15T07:11:21.615Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1601 | 2024-05-15T07:11:21.615Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1602 | 2024-05-15T07:11:21.697Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1603 | 2024-05-15T07:11:21.698Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1604 | 2024-05-15T07:11:21.698Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1605 | 2024-05-15T07:11:21.698Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1606 | 2024-05-15T07:11:21.698Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1607 | 2024-05-15T07:11:21.788Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
file = illumos-utils/src/zone.rs:315
|
1608 | 2024-05-15T07:11:21.793Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
file = illumos-utils/src/zone.rs:315
|
1609 | 2024-05-15T07:11:21.797Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
file = illumos-utils/src/zone.rs:315
|
1610 | 2024-05-15T07:11:21.801Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
file = illumos-utils/src/zone.rs:315
|
1611 | 2024-05-15T07:11:21.806Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
file = illumos-utils/src/zone.rs:315
|
1612 | 2024-05-15T07:11:21.810Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
file = illumos-utils/src/zone.rs:354
|
1613 | 2024-05-15T07:11:21.815Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
file = illumos-utils/src/zone.rs:354
|
1614 | 2024-05-15T07:11:21.819Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
file = illumos-utils/src/zone.rs:354
|
1615 | 2024-05-15T07:11:21.824Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
file = illumos-utils/src/zone.rs:354
|
1616 | 2024-05-15T07:11:21.828Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
file = illumos-utils/src/zone.rs:354
|
1617 | 2024-05-15T07:11:24.586Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService9"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::7"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::7"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1618 | 2024-05-15T07:11:24.586Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1619 | 2024-05-15T07:11:24.617Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService6"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::4"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::4"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1620 | 2024-05-15T07:11:24.618Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1621 | 2024-05-15T07:11:24.670Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService7"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::5"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1622 | 2024-05-15T07:11:24.670Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1623 | 2024-05-15T07:11:24.700Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService8"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::6"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::6"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1624 | 2024-05-15T07:11:24.700Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1625 | 2024-05-15T07:11:24.707Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService5"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::3"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::3"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1626 | 2024-05-15T07:11:24.707Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1627 | 2024-05-15T07:11:24.739Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364") failed: Property not found. retry in 25.957277ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1628 | 2024-05-15T07:11:24.760Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 30.158492ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1629 | 2024-05-15T07:11:24.774Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364") failed: Property not found. retry in 98.91011ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1630 | 2024-05-15T07:11:24.799Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 110.053605ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1631 | 2024-05-15T07:11:24.830Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14") failed: Property not found. retry in 26.200463ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1632 | 2024-05-15T07:11:24.874Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14") failed: Property not found. retry in 118.247142ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1633 | 2024-05-15T07:11:24.890Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364") failed: Property not found. retry in 208.97809ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1634 | 2024-05-15T07:11:25.002Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 126.928496ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1635 | 2024-05-15T07:11:25.949Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015") failed: Property not found. retry in 55.062582ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1636 | 2024-05-15T07:11:26.008Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14") failed: Property not found. retry in 255.533843ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1637 | 2024-05-15T07:11:26.656Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3") failed: Property not found. retry in 62.558119ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1638 | 2024-05-15T07:11:27.235Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364") failed: Property not found. retry in 341.651539ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1639 | 2024-05-15T07:11:27.450Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015") failed: Property not found. retry in 140.64136ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1640 | 2024-05-15T07:11:27.501Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 316.493892ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1641 | 2024-05-15T07:11:27.518Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14") failed: Property not found. retry in 538.016106ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1642 | 2024-05-15T07:11:27.534Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3") failed: Property not found. retry in 103.732415ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1643 | 2024-05-15T07:11:27.589Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364") failed: Property not found. retry in 658.982117ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1644 | 2024-05-15T07:11:27.602Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015") failed: Property not found. retry in 158.318981ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1645 | 2024-05-15T07:11:27.649Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3") failed: Property not found. retry in 218.258222ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1646 | 2024-05-15T07:11:27.772Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015") failed: Property not found. retry in 310.254095ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1647 | 2024-05-15T07:11:27.828Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 563.716868ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1648 | 2024-05-15T07:11:27.877Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3") failed: Property not found. retry in 314.45902ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1649 | 2024-05-15T07:11:28.069Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14") failed: Property not found. retry in 749.94118ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1650 | 2024-05-15T07:11:28.093Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015") failed: Property not found. retry in 822.716309ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1651 | 2024-05-15T07:11:28.202Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3") failed: Property not found. retry in 433.775582ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1652 | 2024-05-15T07:11:28.255Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364") failed: Property not found. retry in 1.259350939s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1653 | 2024-05-15T07:11:28.399Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 1.472003621s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1654 | 2024-05-15T07:11:28.644Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3") failed: Property not found. retry in 672.026583ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1655 | 2024-05-15T07:11:28.826Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14") failed: Property not found. retry in 1.370054446s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1656 | 2024-05-15T07:11:28.923Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015") failed: Property not found. retry in 841.249836ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1657 | 2024-05-15T07:11:29.323Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3") failed: Property not found. retry in 937.460185ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1658 | 2024-05-15T07:11:29.522Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364") failed: Property not found. retry in 1.35119037s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1659 | 2024-05-15T07:11:29.771Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015") failed: Property not found. retry in 557.166031ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1660 | 2024-05-15T07:11:29.875Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 1.337678125s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1661 | 2024-05-15T07:11:30.203Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14") failed: Property not found. retry in 1.368348792s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1662 | 2024-05-15T07:11:30.267Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3") failed: Property not found. retry in 1.042648909s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1663 | 2024-05-15T07:11:30.335Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015") failed: Property not found. retry in 1.205500267s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1664 | 2024-05-15T07:11:30.878Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364") failed: Property not found. retry in 952.148133ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1665 | 2024-05-15T07:11:31.218Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 1.159055229s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1666 | 2024-05-15T07:11:31.315Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3") failed: Property not found. retry in 1.435099279s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1667 | 2024-05-15T07:11:31.547Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015") failed: Property not found. retry in 676.030687ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1668 | 2024-05-15T07:11:31.576Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14") failed: Property not found. retry in 1.215315155s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1669 | 2024-05-15T07:11:31.837Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364") failed: Property not found. retry in 1.265614948s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1670 | 2024-05-15T07:11:32.231Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015") failed: Property not found. retry in 584.997008ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1671 | 2024-05-15T07:11:32.384Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 1.253841954s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1672 | 2024-05-15T07:11:32.756Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3") failed: Property not found. retry in 1.148210669s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1673 | 2024-05-15T07:11:32.798Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14") failed: Property not found. retry in 578.659749ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1674 | 2024-05-15T07:11:32.825Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015") failed: Property not found. retry in 500.754411ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1675 | 2024-05-15T07:11:33.109Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364") failed: Property not found. retry in 1.189335243s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1676 | 2024-05-15T07:11:33.169Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:55222
|
1677 | 2024-05-15T07:11:33.169Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 73
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:55222
req_id = 811fb532-33e0-40c4-a521-fef1ba63c7b2
response_code = 200
uri = /baseboard
|
1678 | 2024-05-15T07:11:33.335Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015") failed: Property not found. retry in 1.417304303s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1679 | 2024-05-15T07:11:33.386Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14") failed: Property not found. retry in 653.437501ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1680 | 2024-05-15T07:11:33.648Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 562.716695ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1681 | 2024-05-15T07:11:33.913Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3") failed: Property not found. retry in 919.566228ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1682 | 2024-05-15T07:11:34.048Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14") failed: Property not found. retry in 1.361696063s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1683 | 2024-05-15T07:11:34.218Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 671.668595ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1684 | 2024-05-15T07:11:34.302Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364") failed: Property not found. retry in 1.03296041s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1685 | 2024-05-15T07:11:34.758Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015") failed: Property not found. retry in 1.325834561s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1686 | 2024-05-15T07:11:34.838Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3") failed: Property not found. retry in 1.399986764s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1687 | 2024-05-15T07:11:34.896Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 662.064363ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1688 | 2024-05-15T07:11:35.341Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364") failed: Property not found. retry in 1.292759464s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1689 | 2024-05-15T07:11:35.417Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14") failed: Property not found. retry in 1.436275437s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1690 | 2024-05-15T07:11:35.568Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 1.449790263s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1691 | 2024-05-15T07:11:36.092Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015") failed: Property not found. retry in 1.366299925s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1692 | 2024-05-15T07:11:36.244Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3") failed: Property not found. retry in 1.156617879s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1693 | 2024-05-15T07:11:36.640Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364") failed: Property not found. retry in 1.063676424s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1694 | 2024-05-15T07:11:36.861Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14") failed: Property not found. retry in 1.413317327s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1695 | 2024-05-15T07:11:37.024Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 1.030118813s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1696 | 2024-05-15T07:11:38.062Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 856.609269ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1697 | 2024-05-15T07:11:38.928Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 1.079181332s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1698 | 2024-05-15T07:11:40.017Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 1.216604685s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1699 | 2024-05-15T07:11:41.915Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 894.910559ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1700 | 2024-05-15T07:11:42.820Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 754.315926ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1701 | 2024-05-15T07:11:43.586Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 715.746732ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1702 | 2024-05-15T07:11:44.311Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 1.099954363s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1703 | 2024-05-15T07:11:45.421Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 786.999032ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1704 | 2024-05-15T07:11:46.218Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 591.426581ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1705 | 2024-05-15T07:11:46.819Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9") failed: Property not found. retry in 756.991228ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1706 | 2024-05-15T07:11:47.597Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015
|
1707 | 2024-05-15T07:11:47.597Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9
|
1708 | 2024-05-15T07:11:47.597Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14
|
1709 | 2024-05-15T07:11:47.597Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3
|
1710 | 2024-05-15T07:11:47.597Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364
|
1711 | 2024-05-15T07:11:47.597Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
1712 | 2024-05-15T07:11:47.598Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
1713 | 2024-05-15T07:11:47.707Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 26295351
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46219
req_id = 985021d9-8241-43da-9ea6-6213bf73e147
response_code = 204
uri = /omicron-zones
|
1714 | 2024-05-15T07:11:47.739Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:54422
|
1715 | 2024-05-15T07:11:47.767Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3437
|
1716 | 2024-05-15T07:11:54.294Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3455
|
1717 | 2024-05-15T07:11:58.073Z | INFO | SledAgent (ServiceManager): Formatting CRDB - Completed
file = sled-agent/src/services.rs:3478
|
1718 | 2024-05-15T07:11:58.073Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 10333757
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:54422
req_id = 0927ca2a-e7e8-4b09-ad02-9dd0b8f66b7d
response_code = 204
uri = /cockroachdb
|
1719 | 2024-05-15T07:11:58.135Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name("ntp-952adea2-114c-4099-9bd9-86f261fe9687"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name("external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e5538333-fcad-480c-9181-aa7e94aaf834, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name("external-dns-e5538333-fcad-480c-9181-aa7e94aaf834"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99861af6-438d-4bef-ba93-632645b46d2f, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name("nexus-99861af6-438d-4bef-ba93-632645b46d2f"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name("nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name("nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7d8b5024-88b0-4173-8867-4e833f05550e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7ce930b4-1bea-41ce-8c9a-2aa16fd16d05, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 638e5dab-76c4-4b78-b15a-a7faf399d1d8, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 42687f02-f9e0-4e2e-8b1e-8fcc2648d21d, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: ae891598-60c4-47ca-bd0b-302a9e60a0bc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7b766ca5-5d0c-459d-bc67-f339bcac986b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9085d-76ce-4798-9d35-12ceecd4c607, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0bae9f0-075a-41e1-a5e1-22133ae05550, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d62b80e0-a2d7-4b05-8ee7-7f6a9516881e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }, OmicronZoneConfig { id: d41b4327-a68c-4bde-a48b-cfcb75b4c054, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5da68ecb-08f6-4895-925f-903c81000441 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecac0828-7786-4503-af9f-835e69870288, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool), kind: External } } } }, OmicronZoneConfig { id: eece7197-e5eb-4531-85f7-6700f07faf6b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool), kind: External } } } }, OmicronZoneConfig { id: c06cd1db-bb07-41fa-af89-0fd52ff0a0a8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:430
sled_address = [fd00:1122:3344:101::1]:12345
|
1720 | 2024-05-15T07:11:58.135Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:60075
|
1721 | 2024-05-15T07:11:58.136Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1722 | 2024-05-15T07:11:58.136Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1723 | 2024-05-15T07:11:58.160Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1724 | 2024-05-15T07:11:58.160Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1725 | 2024-05-15T07:11:58.182Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1726 | 2024-05-15T07:11:58.182Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
1727 | 2024-05-15T07:11:58.202Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1728 | 2024-05-15T07:11:58.203Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1729 | 2024-05-15T07:11:58.223Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1730 | 2024-05-15T07:11:58.223Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1731 | 2024-05-15T07:11:58.244Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1732 | 2024-05-15T07:11:58.244Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1733 | 2024-05-15T07:11:58.264Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1734 | 2024-05-15T07:11:58.264Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1735 | 2024-05-15T07:11:58.284Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1736 | 2024-05-15T07:11:58.284Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }
|
1737 | 2024-05-15T07:11:58.304Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1738 | 2024-05-15T07:11:58.304Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
1739 | 2024-05-15T07:11:58.378Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e5538333-fcad-480c-9181-aa7e94aaf834, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1740 | 2024-05-15T07:11:58.378Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e5538333-fcad-480c-9181-aa7e94aaf834, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
1741 | 2024-05-15T07:11:58.454Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ce930b4-1bea-41ce-8c9a-2aa16fd16d05, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1742 | 2024-05-15T07:11:58.454Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 7ce930b4-1bea-41ce-8c9a-2aa16fd16d05, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
1743 | 2024-05-15T07:11:58.527Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ae891598-60c4-47ca-bd0b-302a9e60a0bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1744 | 2024-05-15T07:11:58.527Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ae891598-60c4-47ca-bd0b-302a9e60a0bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
1745 | 2024-05-15T07:11:58.598Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7b766ca5-5d0c-459d-bc67-f339bcac986b, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1746 | 2024-05-15T07:11:58.598Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 7b766ca5-5d0c-459d-bc67-f339bcac986b, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: Crucible }, responder: ... }
|
1747 | 2024-05-15T07:11:58.667Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 03e9085d-76ce-4798-9d35-12ceecd4c607, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1748 | 2024-05-15T07:11:58.667Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: 03e9085d-76ce-4798-9d35-12ceecd4c607, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
1749 | 2024-05-15T07:11:58.745Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e0bae9f0-075a-41e1-a5e1-22133ae05550, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1750 | 2024-05-15T07:11:58.746Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: e0bae9f0-075a-41e1-a5e1-22133ae05550, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
1751 | 2024-05-15T07:11:58.819Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d62b80e0-a2d7-4b05-8ee7-7f6a9516881e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1752 | 2024-05-15T07:11:58.819Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d62b80e0-a2d7-4b05-8ee7-7f6a9516881e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
1753 | 2024-05-15T07:11:58.894Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d41b4327-a68c-4bde-a48b-cfcb75b4c054, dataset_name: DatasetName { pool_name: ZpoolName { id: 5da68ecb-08f6-4895-925f-903c81000441 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1754 | 2024-05-15T07:11:58.894Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: d41b4327-a68c-4bde-a48b-cfcb75b4c054, dataset_name: DatasetName { pool_name: ZpoolName { id: 5da68ecb-08f6-4895-925f-903c81000441 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
1755 | 2024-05-15T07:11:58.961Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: ecac0828-7786-4503-af9f-835e69870288, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1756 | 2024-05-15T07:11:58.961Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: ecac0828-7786-4503-af9f-835e69870288, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
1757 | 2024-05-15T07:11:59.030Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eece7197-e5eb-4531-85f7-6700f07faf6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1758 | 2024-05-15T07:11:59.030Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: eece7197-e5eb-4531-85f7-6700f07faf6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool), kind: External }, kind: Crucible }, responder: ... }
|
1759 | 2024-05-15T07:11:59.097Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c06cd1db-bb07-41fa-af89-0fd52ff0a0a8, dataset_name: DatasetName { pool_name: ZpoolName { id: bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:381
|
1760 | 2024-05-15T07:11:59.097Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:809
request = NewFilesystemRequest { dataset_id: c06cd1db-bb07-41fa-af89-0fd52ff0a0a8, dataset_name: DatasetName { pool_name: ZpoolName { id: bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
1761 | 2024-05-15T07:11:59.166Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1762 | 2024-05-15T07:11:59.167Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1763 | 2024-05-15T07:11:59.167Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
1764 | 2024-05-15T07:11:59.167Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1765 | 2024-05-15T07:11:59.286Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:259
|
1766 | 2024-05-15T07:11:59.299Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:259
|
1767 | 2024-05-15T07:11:59.341Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:259
|
1768 | 2024-05-15T07:11:59.341Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1769 | 2024-05-15T07:11:59.341Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1770 | 2024-05-15T07:11:59.341Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1771 | 2024-05-15T07:11:59.341Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1772 | 2024-05-15T07:11:59.341Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1773 | 2024-05-15T07:11:59.342Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:259
|
1774 | 2024-05-15T07:11:59.357Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1775 | 2024-05-15T07:11:59.357Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1776 | 2024-05-15T07:11:59.591Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:259
|
1777 | 2024-05-15T07:11:59.591Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1778 | 2024-05-15T07:11:59.591Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1779 | 2024-05-15T07:11:59.591Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1780 | 2024-05-15T07:11:59.591Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1781 | 2024-05-15T07:11:59.591Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1782 | 2024-05-15T07:11:59.591Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1783 | 2024-05-15T07:11:59.591Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1784 | 2024-05-15T07:11:59.592Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
file = illumos-utils/src/zone.rs:315
|
1785 | 2024-05-15T07:11:59.597Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
file = illumos-utils/src/zone.rs:315
|
1786 | 2024-05-15T07:11:59.602Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
file = illumos-utils/src/zone.rs:315
|
1787 | 2024-05-15T07:11:59.607Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
file = illumos-utils/src/zone.rs:315
|
1788 | 2024-05-15T07:11:59.612Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
file = illumos-utils/src/zone.rs:315
|
1789 | 2024-05-15T07:11:59.618Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
file = illumos-utils/src/zone.rs:315
|
1790 | 2024-05-15T07:11:59.623Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
file = illumos-utils/src/zone.rs:315
|
1791 | 2024-05-15T07:11:59.774Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
file = illumos-utils/src/zone.rs:354
|
1792 | 2024-05-15T07:11:59.779Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
file = illumos-utils/src/zone.rs:354
|
1793 | 2024-05-15T07:11:59.784Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
file = illumos-utils/src/zone.rs:354
|
1794 | 2024-05-15T07:11:59.788Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
file = illumos-utils/src/zone.rs:354
|
1795 | 2024-05-15T07:11:59.793Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
file = illumos-utils/src/zone.rs:354
|
1796 | 2024-05-15T07:11:59.798Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
file = illumos-utils/src/zone.rs:354
|
1797 | 2024-05-15T07:11:59.803Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
file = illumos-utils/src/zone.rs:354
|
1798 | 2024-05-15T07:11:59.808Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
file = illumos-utils/src/zone.rs:315
|
1799 | 2024-05-15T07:11:59.814Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
file = illumos-utils/src/zone.rs:315
|
1800 | 2024-05-15T07:11:59.819Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
file = illumos-utils/src/zone.rs:315
|
1801 | 2024-05-15T07:11:59.824Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
file = illumos-utils/src/zone.rs:315
|
1802 | 2024-05-15T07:11:59.831Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
file = illumos-utils/src/zone.rs:315
|
1803 | 2024-05-15T07:11:59.837Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
file = illumos-utils/src/zone.rs:315
|
1804 | 2024-05-15T07:11:59.844Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
file = illumos-utils/src/zone.rs:315
|
1805 | 2024-05-15T07:11:59.851Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
file = illumos-utils/src/zone.rs:354
|
1806 | 2024-05-15T07:11:59.856Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
file = illumos-utils/src/zone.rs:354
|
1807 | 2024-05-15T07:11:59.862Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
file = illumos-utils/src/zone.rs:354
|
1808 | 2024-05-15T07:11:59.868Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
file = illumos-utils/src/zone.rs:354
|
1809 | 2024-05-15T07:11:59.874Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
file = illumos-utils/src/zone.rs:354
|
1810 | 2024-05-15T07:11:59.880Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
file = illumos-utils/src/zone.rs:354
|
1811 | 2024-05-15T07:11:59.887Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
file = illumos-utils/src/zone.rs:354
|
1812 | 2024-05-15T07:11:59.946Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:276
|
1813 | 2024-05-15T07:12:00.050Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:276
|
1814 | 2024-05-15T07:12:00.163Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:276
|
1815 | 2024-05-15T07:12:00.220Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:276
|
1816 | 2024-05-15T07:12:00.225Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:301
|
1817 | 2024-05-15T07:12:00.225Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:193
|
1818 | 2024-05-15T07:12:00.353Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte1", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 222, 14]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte1" } }
|
1819 | 2024-05-15T07:12:00.353Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = nexus
|
1820 | 2024-05-15T07:12:00.356Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:301
|
1821 | 2024-05-15T07:12:00.356Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:193
|
1822 | 2024-05-15T07:12:00.464Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte2", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 234, 140]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte2" } }
|
1823 | 2024-05-15T07:12:00.464Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = nexus
|
1824 | 2024-05-15T07:12:00.520Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:276
|
1825 | 2024-05-15T07:12:00.522Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:301
|
1826 | 2024-05-15T07:12:00.522Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:193
|
1827 | 2024-05-15T07:12:00.629Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte3", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 204, 142]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: "vopte3" } }
|
1828 | 2024-05-15T07:12:00.629Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = external_dns
|
1829 | 2024-05-15T07:12:00.634Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:301
|
1830 | 2024-05-15T07:12:00.634Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:193
|
1831 | 2024-05-15T07:12:00.769Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte4", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 254, 70]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: "vopte4" } }
|
1832 | 2024-05-15T07:12:00.769Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = external_dns
|
1833 | 2024-05-15T07:12:00.771Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
1834 | 2024-05-15T07:12:00.771Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
1835 | 2024-05-15T07:12:00.774Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
1836 | 2024-05-15T07:12:00.774Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
1837 | 2024-05-15T07:12:00.777Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:301
|
1838 | 2024-05-15T07:12:00.777Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:193
|
1839 | 2024-05-15T07:12:00.865Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte5", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 207, 1]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte5" } }
|
1840 | 2024-05-15T07:12:00.865Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1290
zone_type = nexus
|
1841 | 2024-05-15T07:12:00.866Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
1842 | 2024-05-15T07:12:00.866Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
1843 | 2024-05-15T07:12:00.866Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
1844 | 2024-05-15T07:12:00.866Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
1845 | 2024-05-15T07:12:00.866Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
1846 | 2024-05-15T07:12:00.869Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
1847 | 2024-05-15T07:12:00.872Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1848 | 2024-05-15T07:12:00.872Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1849 | 2024-05-15T07:12:00.872Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
1850 | 2024-05-15T07:12:00.874Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
1851 | 2024-05-15T07:12:00.877Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1852 | 2024-05-15T07:12:00.877Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1853 | 2024-05-15T07:12:01.042Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
file = illumos-utils/src/zone.rs:315
|
1854 | 2024-05-15T07:12:01.051Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
file = illumos-utils/src/zone.rs:315
|
1855 | 2024-05-15T07:12:01.061Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:102
|
1856 | 2024-05-15T07:12:01.061Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:109
|
1857 | 2024-05-15T07:12:01.061Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
file = illumos-utils/src/zone.rs:315
|
1858 | 2024-05-15T07:12:01.071Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
file = illumos-utils/src/zone.rs:315
|
1859 | 2024-05-15T07:12:01.080Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:137
|
1860 | 2024-05-15T07:12:01.083Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
1861 | 2024-05-15T07:12:01.127Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
file = illumos-utils/src/zone.rs:354
|
1862 | 2024-05-15T07:12:01.136Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
file = illumos-utils/src/zone.rs:354
|
1863 | 2024-05-15T07:12:01.146Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
file = illumos-utils/src/zone.rs:354
|
1864 | 2024-05-15T07:12:01.155Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
file = illumos-utils/src/zone.rs:354
|
1865 | 2024-05-15T07:12:01.164Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
file = illumos-utils/src/zone.rs:315
|
1866 | 2024-05-15T07:12:01.218Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
file = illumos-utils/src/zone.rs:354
|
1867 | 2024-05-15T07:12:03.456Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:42916
|
1868 | 2024-05-15T07:12:03.461Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 5974
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:42916
req_id = ac69487d-a473-4478-b014-8079d6ea2a84
response_code = 200
uri = /baseboard
|
1869 | 2024-05-15T07:12:05.216Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService16"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::10"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::10"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1870 | 2024-05-15T07:12:05.232Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
1871 | 2024-05-15T07:12:05.243Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService13"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::14"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_08d4507c-e72e-4afd-9f03-5e28f4ad0909/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::14"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="3a0d11c9-b2a2-4a79-876b-36faa9fb4d9f"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1872 | 2024-05-15T07:12:05.246Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
1873 | 2024-05-15T07:12:05.265Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService12"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::13"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_12d428b9-eedd-40c0-8928-1d92a73f181d/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::13"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="17ecf68f-f3fc-4c24-b9d0-9bbad0bb0f5f"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1874 | 2024-05-15T07:12:05.266Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
1875 | 2024-05-15T07:12:05.312Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService11"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::f"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::f"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1876 | 2024-05-15T07:12:05.313Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
1877 | 2024-05-15T07:12:05.574Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService20"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::11"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::11"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1878 | 2024-05-15T07:12:05.574Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
1879 | 2024-05-15T07:12:05.716Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService19"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::1a"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_bae019e4-4e95-466e-aa8a-f660660e26c1/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::1a"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="8e2c2ebf-8081-4938-81e8-8c81b0b67a2f"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1880 | 2024-05-15T07:12:05.716Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
1881 | 2024-05-15T07:12:05.736Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService17"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::17"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_5da68ecb-08f6-4895-925f-903c81000441/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::17"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="979acc2a-4937-4b2a-9089-a4c17553a937"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1882 | 2024-05-15T07:12:05.738Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
1883 | 2024-05-15T07:12:05.758Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService22"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::15"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_7ba4af02-bf8f-47c6-af79-d69fdad5d4bf/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::15"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="089c3f0c-afa2-449f-8ea8-924037b9db1f"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1884 | 2024-05-15T07:12:05.760Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
1885 | 2024-05-15T07:12:05.846Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607") failed: Property not found. retry in 70.913775ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
1886 | 2024-05-15T07:12:05.881Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 58.201696ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
1887 | 2024-05-15T07:12:05.909Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b") failed: Property not found. retry in 65.088482ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
1888 | 2024-05-15T07:12:05.942Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d") failed: Property not found. retry in 35.321707ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
1889 | 2024-05-15T07:12:05.945Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService18"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::19"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_2780ea55-1799-4c3e-916a-9024b9586eaf/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::19"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="36afc74d-4c2b-4c9a-aec0-794dbba4f078"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1890 | 2024-05-15T07:12:05.971Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 109.563332ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
1891 | 2024-05-15T07:12:06.093Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607") failed: Property not found. retry in 98.091036ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
1892 | 2024-05-15T07:12:06.094Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService14"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::16"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_62a187a2-0706-420e-a881-83ff781ad6c1/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::16"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="5b5fcf9b-c81b-46f9-b73e-97c320d093b8"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1893 | 2024-05-15T07:12:06.134Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d") failed: Property not found. retry in 57.054824ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
1894 | 2024-05-15T07:12:06.134Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
1895 | 2024-05-15T07:12:06.200Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_ecac0828-7786-4503-af9f-835e69870288:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService21"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::18"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_3b36f3f1-29d0-4d11-8b6b-ae2a89620d43/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::18"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="4a530305-da63-4a10-bd54-fea68adbdc01"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1896 | 2024-05-15T07:12:06.232Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 286.166615ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
1897 | 2024-05-15T07:12:06.262Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b") failed: Property not found. retry in 124.811779ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
1898 | 2024-05-15T07:12:06.300Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607") failed: Property not found. retry in 288.701945ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
1899 | 2024-05-15T07:12:06.301Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
1900 | 2024-05-15T07:12:06.327Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService15"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::12"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value="oxp_4e6ef30f-a7d9-4acc-99aa-59ae1084fa14/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::12"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="0f7f620d-0605-4161-b644-f2b21242487b"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1901 | 2024-05-15T07:12:06.377Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d") failed: Property not found. retry in 249.772146ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
1902 | 2024-05-15T07:12:06.465Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 36.638939ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
1903 | 2024-05-15T07:12:06.579Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 67.040375ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
1904 | 2024-05-15T07:12:06.610Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a") failed: Property not found. retry in 48.163472ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
1905 | 2024-05-15T07:12:06.610Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
1906 | 2024-05-15T07:12:06.659Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 61.501736ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
1907 | 2024-05-15T07:12:06.691Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 490.920959ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
1908 | 2024-05-15T07:12:06.723Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b") failed: Property not found. retry in 285.551762ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
1909 | 2024-05-15T07:12:06.752Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607") failed: Property not found. retry in 321.40866ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
1910 | 2024-05-15T07:12:06.753Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
1911 | 2024-05-15T07:12:06.804Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d") failed: Property not found. retry in 527.986927ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
1912 | 2024-05-15T07:12:06.844Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 63.01472ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
1913 | 2024-05-15T07:12:06.875Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 31.057474ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
1914 | 2024-05-15T07:12:06.924Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 134.427809ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
1915 | 2024-05-15T07:12:06.969Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a") failed: Property not found. retry in 50.379648ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
1916 | 2024-05-15T07:12:07.021Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 91.018381ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
1917 | 2024-05-15T07:12:07.087Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b") failed: Property not found. retry in 442.077016ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
1918 | 2024-05-15T07:12:07.102Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607") failed: Property not found. retry in 828.884504ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
1919 | 2024-05-15T07:12:07.117Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 40.856196ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
1920 | 2024-05-15T07:12:07.134Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 147.274322ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
1921 | 2024-05-15T07:12:07.150Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 61.250375ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
1922 | 2024-05-15T07:12:07.166Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 213.765742ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
1923 | 2024-05-15T07:12:07.183Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a") failed: Property not found. retry in 199.533726ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
1924 | 2024-05-15T07:12:07.197Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 58.902693ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
1925 | 2024-05-15T07:12:07.212Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 130.59219ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
1926 | 2024-05-15T07:12:07.225Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 480.695523ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
1927 | 2024-05-15T07:12:07.237Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 79.090983ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
1928 | 2024-05-15T07:12:07.276Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 264.09353ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
1929 | 2024-05-15T07:12:07.333Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 88.294372ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
1930 | 2024-05-15T07:12:07.333Z | INFO | SledAgent (ServiceManager): Profile for oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService10"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::d"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/oximeter">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="id" value="7d8b5024-88b0-4173-8867-4e833f05550e"/>
<propval type="astring" name="address" value="[fd00:1122:3344:101::d]:12223"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1931 | 2024-05-15T07:12:07.347Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 101.591147ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
1932 | 2024-05-15T07:12:07.367Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 29.728007ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
1933 | 2024-05-15T07:12:07.391Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d") failed: Property not found. retry in 843.951732ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
1934 | 2024-05-15T07:12:07.408Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 240.012902ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
1935 | 2024-05-15T07:12:07.442Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 540.691348ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
1936 | 2024-05-15T07:12:07.498Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a") failed: Property not found. retry in 378.626418ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
1937 | 2024-05-15T07:12:07.529Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 160.296603ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
1938 | 2024-05-15T07:12:07.553Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 228.176797ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
1939 | 2024-05-15T07:12:07.554Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
1940 | 2024-05-15T07:12:07.592Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b") failed: Property not found. retry in 611.97574ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
1941 | 2024-05-15T07:12:07.626Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 550.99456ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
1942 | 2024-05-15T07:12:07.648Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 127.664502ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
1943 | 2024-05-15T07:12:07.669Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 207.599915ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
1944 | 2024-05-15T07:12:07.693Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 800.725519ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
1945 | 2024-05-15T07:12:07.713Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 208.371936ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
1946 | 2024-05-15T07:12:07.732Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 1.063199578s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
1947 | 2024-05-15T07:12:07.802Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 145.467796ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
1948 | 2024-05-15T07:12:07.819Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 1.050813951s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
1949 | 2024-05-15T07:12:07.926Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 565.227514ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
1950 | 2024-05-15T07:12:08.012Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 28.217488ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
1951 | 2024-05-15T07:12:08.040Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a") failed: Property not found. retry in 444.229363ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
1952 | 2024-05-15T07:12:08.072Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 838.698203ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
1953 | 2024-05-15T07:12:08.093Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 145.201977ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
1954 | 2024-05-15T07:12:08.111Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607") failed: Property not found. retry in 1.349334644s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
1955 | 2024-05-15T07:12:08.127Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 251.399211ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
1956 | 2024-05-15T07:12:08.145Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 478.35567ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
1957 | 2024-05-15T07:12:08.198Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 859.331364ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
1958 | 2024-05-15T07:12:08.224Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b") failed: Property not found. retry in 1.063639161s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
1959 | 2024-05-15T07:12:08.253Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d") failed: Property not found. retry in 1.039199175s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
1960 | 2024-05-15T07:12:08.271Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 160.392533ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
1961 | 2024-05-15T07:12:08.400Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 721.282628ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
1962 | 2024-05-15T07:12:08.497Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 441.426171ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
1963 | 2024-05-15T07:12:08.498Z | INFO | SledAgent (ServiceManager): Profile for oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService26"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::9"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte3"/>
<propval type="astring" name="gateway" value="172.30.1.1"/>
<propval type="astring" name="ip" value="172.30.1.6"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/external_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:101::9]:5353"/>
<propval type="astring" name="dns_address" value="172.30.1.6:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1964 | 2024-05-15T07:12:08.500Z | INFO | SledAgent (ServiceManager): Profile for oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService27"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::8"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte4"/>
<propval type="astring" name="gateway" value="172.30.1.1"/>
<propval type="astring" name="ip" value="172.30.1.5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/external_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value="[fd00:1122:3344:101::8]:5353"/>
<propval type="astring" name="dns_address" value="172.30.1.5:53"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1965 | 2024-05-15T07:12:08.521Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 1.028210724s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
1966 | 2024-05-15T07:12:08.539Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 857.969589ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
1967 | 2024-05-15T07:12:08.557Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a") failed: Property not found. retry in 1.453556572s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
1968 | 2024-05-15T07:12:08.557Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
1969 | 2024-05-15T07:12:08.564Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
1970 | 2024-05-15T07:12:08.645Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 822.642841ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
1971 | 2024-05-15T07:12:08.812Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 1.015358733s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
1972 | 2024-05-15T07:12:08.883Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 44.48971ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
1973 | 2024-05-15T07:12:08.897Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 68.036118ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
1974 | 2024-05-15T07:12:08.912Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 906.113497ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
1975 | 2024-05-15T07:12:08.930Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 950.267308ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
1976 | 2024-05-15T07:12:08.945Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 95.580084ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
1977 | 2024-05-15T07:12:08.963Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 749.823705ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
1978 | 2024-05-15T07:12:09.025Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 53.944077ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
1979 | 2024-05-15T07:12:09.056Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 240.49042ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
1980 | 2024-05-15T07:12:09.071Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 1.151980058s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
1981 | 2024-05-15T07:12:09.090Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 233.125045ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
1982 | 2024-05-15T07:12:09.137Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 1.382385141s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
1983 | 2024-05-15T07:12:09.331Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b") failed: Property not found. retry in 997.517618ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
1984 | 2024-05-15T07:12:09.356Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 443.243693ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
1985 | 2024-05-15T07:12:09.383Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 475.133711ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
1986 | 2024-05-15T07:12:09.402Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d") failed: Property not found. retry in 684.03519ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
1987 | 2024-05-15T07:12:09.418Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 760.131343ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
1988 | 2024-05-15T07:12:09.481Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607") failed: Property not found. retry in 1.176744391s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
1989 | 2024-05-15T07:12:09.499Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 925.496001ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
1990 | 2024-05-15T07:12:09.566Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 1.369378606s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
1991 | 2024-05-15T07:12:09.692Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService25"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::b"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte2"/>
<propval type="astring" name="gateway" value="172.30.2.1"/>
<propval type="astring" name="ip" value="172.30.2.6"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1992 | 2024-05-15T07:12:09.693Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService24"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::a"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte1"/>
<propval type="astring" name="gateway" value="172.30.2.1"/>
<propval type="astring" name="ip" value="172.30.2.5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1993 | 2024-05-15T07:12:09.693Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService28"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::c"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value="vopte5"/>
<propval type="astring" name="gateway" value="172.30.2.1"/>
<propval type="astring" name="ip" value="172.30.2.7"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
1994 | 2024-05-15T07:12:09.693Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
1995 | 2024-05-15T07:12:09.703Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
1996 | 2024-05-15T07:12:09.711Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
1997 | 2024-05-15T07:12:09.738Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 1.372029068s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
1998 | 2024-05-15T07:12:09.822Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 401.029705ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
1999 | 2024-05-15T07:12:09.840Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 1.172856053s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
2000 | 2024-05-15T07:12:09.861Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 933.29296ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
2001 | 2024-05-15T07:12:09.891Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 789.186134ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
2002 | 2024-05-15T07:12:09.935Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 809.355217ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
2003 | 2024-05-15T07:12:10.037Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a") failed: Property not found. retry in 1.082991259s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
2004 | 2024-05-15T07:12:10.079Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 56.129409ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2005 | 2024-05-15T07:12:10.107Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d") failed: Property not found. retry in 1.162919272s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
2006 | 2024-05-15T07:12:10.122Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959") failed: Property not found. retry in 48.491294ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2007 | 2024-05-15T07:12:10.138Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3") failed: Property not found. retry in 60.223385ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
2008 | 2024-05-15T07:12:10.254Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 54.990004ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2009 | 2024-05-15T07:12:10.472Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 1.433055217s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
2010 | 2024-05-15T07:12:10.531Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b") failed: Property not found. retry in 1.486001911s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
2011 | 2024-05-15T07:12:10.587Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 1.364949824s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
2012 | 2024-05-15T07:12:10.609Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 108.476781ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2013 | 2024-05-15T07:12:10.650Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959") failed: Property not found. retry in 52.780146ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2014 | 2024-05-15T07:12:10.667Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 680.299636ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
2015 | 2024-05-15T07:12:10.683Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 968.209085ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
2016 | 2024-05-15T07:12:10.698Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 645.662698ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
2017 | 2024-05-15T07:12:10.715Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3") failed: Property not found. retry in 59.522041ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
2018 | 2024-05-15T07:12:10.731Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 1.090361916s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
2019 | 2024-05-15T07:12:10.746Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607") failed: Property not found. retry in 853.412333ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
2020 | 2024-05-15T07:12:10.761Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 286.011784ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2021 | 2024-05-15T07:12:10.775Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959") failed: Property not found. retry in 279.648672ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2022 | 2024-05-15T07:12:10.790Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 927.985831ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
2023 | 2024-05-15T07:12:10.804Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3") failed: Property not found. retry in 297.862206ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
2024 | 2024-05-15T07:12:10.818Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 860.700389ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
2025 | 2024-05-15T07:12:10.955Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 914.660657ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
2026 | 2024-05-15T07:12:11.027Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 1.255970183s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
2027 | 2024-05-15T07:12:11.061Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 1.073331655s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2028 | 2024-05-15T07:12:11.074Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959") failed: Property not found. retry in 548.942711ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2029 | 2024-05-15T07:12:11.114Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3") failed: Property not found. retry in 329.540523ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
2030 | 2024-05-15T07:12:11.125Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 881.507601ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
2031 | 2024-05-15T07:12:11.136Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a") failed: Property not found. retry in 624.614174ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
2032 | 2024-05-15T07:12:11.282Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d") failed: Property not found. retry in 1.022325299s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
2033 | 2024-05-15T07:12:11.357Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 513.093397ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
2034 | 2024-05-15T07:12:11.369Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 1.050707225s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
2035 | 2024-05-15T07:12:11.457Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3") failed: Property not found. retry in 709.201981ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
2036 | 2024-05-15T07:12:11.613Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607") failed: Property not found. retry in 1.347231978s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
2037 | 2024-05-15T07:12:11.635Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959") failed: Property not found. retry in 776.032126ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2038 | 2024-05-15T07:12:11.663Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 1.234694938s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
2039 | 2024-05-15T07:12:11.693Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 629.795621ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
2040 | 2024-05-15T07:12:11.730Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 1.041026489s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
2041 | 2024-05-15T07:12:11.772Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a") failed: Property not found. retry in 643.569305ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
2042 | 2024-05-15T07:12:11.834Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 1.110329784s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
2043 | 2024-05-15T07:12:11.881Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 738.081824ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
2044 | 2024-05-15T07:12:11.893Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 586.745915ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
2045 | 2024-05-15T07:12:11.916Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 1.328108368s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
2046 | 2024-05-15T07:12:11.963Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 616.691972ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
2047 | 2024-05-15T07:12:12.019Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 1.378920914s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
2048 | 2024-05-15T07:12:12.029Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b") failed: Property not found. retry in 1.062282019s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
2049 | 2024-05-15T07:12:12.152Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 589.245503ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2050 | 2024-05-15T07:12:12.179Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3") failed: Property not found. retry in 1.415786126s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
2051 | 2024-05-15T07:12:12.297Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 1.013116895s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
2052 | 2024-05-15T07:12:12.373Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 539.704119ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
2053 | 2024-05-15T07:12:12.385Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d") failed: Property not found. retry in 901.398021ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
2054 | 2024-05-15T07:12:12.436Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959") failed: Property not found. retry in 1.033857451s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2055 | 2024-05-15T07:12:12.447Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 1.082133849s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
2056 | 2024-05-15T07:12:12.458Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a") failed: Property not found. retry in 1.385040595s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
2057 | 2024-05-15T07:12:12.492Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 603.732884ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
2058 | 2024-05-15T07:12:12.596Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 546.905954ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
2059 | 2024-05-15T07:12:12.640Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 798.19698ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
2060 | 2024-05-15T07:12:12.652Z | INFO | SledAgent (ServiceManager): Profile for oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService23"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::e"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/clickhouse">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::e"/>
<propval type="astring" name="listen_port" value="8123"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
2061 | 2024-05-15T07:12:12.653Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
|
2062 | 2024-05-15T07:12:12.756Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 951.509426ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2063 | 2024-05-15T07:12:12.784Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 746.175243ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
2064 | 2024-05-15T07:12:12.899Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05") failed: Property not found. retry in 58.561598ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
|
2065 | 2024-05-15T07:12:12.914Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 708.802853ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
2066 | 2024-05-15T07:12:12.927Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 1.432240083s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
2067 | 2024-05-15T07:12:12.959Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 718.649724ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
2068 | 2024-05-15T07:12:12.971Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05") failed: Property not found. retry in 116.051063ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
|
2069 | 2024-05-15T07:12:12.985Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607") failed: Property not found. retry in 696.004218ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
2070 | 2024-05-15T07:12:13.122Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05") failed: Property not found. retry in 111.495997ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
|
2071 | 2024-05-15T07:12:13.164Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b") failed: Property not found. retry in 1.186057807s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
2072 | 2024-05-15T07:12:13.207Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 1.15870666s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
2073 | 2024-05-15T07:12:13.222Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 802.699278ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
2074 | 2024-05-15T07:12:13.332Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05") failed: Property not found. retry in 504.484904ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
|
2075 | 2024-05-15T07:12:13.341Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 86.809086607s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out"
file = oximeter/producer/src/lib.rs:426
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
2076 | 2024-05-15T07:12:13.349Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 1.423001353s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
2077 | 2024-05-15T07:12:13.363Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d") failed: Property not found. retry in 658.339003ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
2078 | 2024-05-15T07:12:13.376Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 1.371182126s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
2079 | 2024-05-15T07:12:13.411Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 1.150556605s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
2080 | 2024-05-15T07:12:13.452Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 1.149615517s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
2081 | 2024-05-15T07:12:13.480Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959") failed: Property not found. retry in 631.371466ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2082 | 2024-05-15T07:12:13.540Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 1.403064319s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
2083 | 2024-05-15T07:12:13.551Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 920.807947ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
2084 | 2024-05-15T07:12:13.605Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3") failed: Property not found. retry in 1.058413597s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
2085 | 2024-05-15T07:12:13.633Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 864.159365ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
2086 | 2024-05-15T07:12:13.689Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 1.277260488s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
2087 | 2024-05-15T07:12:13.700Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607") failed: Property not found. retry in 1.161489836s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
2088 | 2024-05-15T07:12:13.719Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 558.418778ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2089 | 2024-05-15T07:12:13.849Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05") failed: Property not found. retry in 1.053623966s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
|
2090 | 2024-05-15T07:12:13.863Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a") failed: Property not found. retry in 1.308004939s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
2091 | 2024-05-15T07:12:14.039Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d") failed: Property not found. retry in 1.179343084s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
2092 | 2024-05-15T07:12:14.055Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 845.634618ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
2093 | 2024-05-15T07:12:14.127Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959") failed: Property not found. retry in 991.24235ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2094 | 2024-05-15T07:12:14.304Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 906.538841ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2095 | 2024-05-15T07:12:14.377Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b") failed: Property not found. retry in 1.152650042s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
2096 | 2024-05-15T07:12:14.400Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 629.000463ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
2097 | 2024-05-15T07:12:14.428Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 1.40248341s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
2098 | 2024-05-15T07:12:14.494Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 1.464903022s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
2099 | 2024-05-15T07:12:14.538Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 643.901069ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
2100 | 2024-05-15T07:12:14.584Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 1.090909125s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
2101 | 2024-05-15T07:12:14.617Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 1.043662349s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
2102 | 2024-05-15T07:12:14.682Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3") failed: Property not found. retry in 1.008319442s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
2103 | 2024-05-15T07:12:14.766Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 1.282779159s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
2104 | 2024-05-15T07:12:14.796Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 636.933288ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
2105 | 2024-05-15T07:12:14.880Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607") failed: Property not found. retry in 1.022063988s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
2106 | 2024-05-15T07:12:14.922Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 1.150423272s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
2107 | 2024-05-15T07:12:14.942Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05") failed: Property not found. retry in 1.453159607s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
|
2108 | 2024-05-15T07:12:14.962Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 708.472654ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
2109 | 2024-05-15T07:12:14.984Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 759.783325ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
2110 | 2024-05-15T07:12:15.044Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 528.256604ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
2111 | 2024-05-15T07:12:15.133Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959") failed: Property not found. retry in 1.417235023s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2112 | 2024-05-15T07:12:15.191Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a") failed: Property not found. retry in 990.520707ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
2113 | 2024-05-15T07:12:15.205Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 1.472801124s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
2114 | 2024-05-15T07:12:15.222Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 797.24176ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2115 | 2024-05-15T07:12:15.236Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d") failed: Property not found. retry in 1.496639761s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
2116 | 2024-05-15T07:12:15.450Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 1.353067187s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
2117 | 2024-05-15T07:12:15.543Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b") failed: Property not found. retry in 1.159405902s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
2118 | 2024-05-15T07:12:15.590Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 915.292997ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
2119 | 2024-05-15T07:12:15.675Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 734.952931ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
2120 | 2024-05-15T07:12:15.689Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 844.183427ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
2121 | 2024-05-15T07:12:15.703Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 1.439051992s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
2122 | 2024-05-15T07:12:15.718Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3") failed: Property not found. retry in 1.497972851s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
2123 | 2024-05-15T07:12:15.756Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 1.427882038s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
2124 | 2024-05-15T07:12:15.841Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 1.331791726s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
2125 | 2024-05-15T07:12:15.916Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607") failed: Property not found. retry in 1.304453859s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
2126 | 2024-05-15T07:12:15.974Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 991.347004ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
2127 | 2024-05-15T07:12:16.031Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 1.437009393s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2128 | 2024-05-15T07:12:16.059Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 1.428543717s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
2129 | 2024-05-15T07:12:16.088Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 615.056511ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
2130 | 2024-05-15T07:12:16.197Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a") failed: Property not found. retry in 1.216011795s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
2131 | 2024-05-15T07:12:16.409Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05") failed: Property not found. retry in 643.465493ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
|
2132 | 2024-05-15T07:12:16.421Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 977.222651ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
2133 | 2024-05-15T07:12:16.520Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8") failed: Property not found. retry in 706.421811ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
2134 | 2024-05-15T07:12:16.545Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 818.545068ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
2135 | 2024-05-15T07:12:16.569Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959") failed: Property not found. retry in 1.045946717s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2136 | 2024-05-15T07:12:16.691Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 1.102909961s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
2137 | 2024-05-15T07:12:16.715Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b") failed: Property not found. retry in 614.670761ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
2138 | 2024-05-15T07:12:16.727Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 889.513972ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
2139 | 2024-05-15T07:12:16.745Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d") failed: Property not found. retry in 828.220259ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
2140 | 2024-05-15T07:12:16.816Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 948.520622ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
2141 | 2024-05-15T07:12:16.977Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 582.662369ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
2142 | 2024-05-15T07:12:17.067Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05") failed: Property not found. retry in 943.743124ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
|
2143 | 2024-05-15T07:12:17.158Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 1.032182523s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
2144 | 2024-05-15T07:12:17.190Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 1.036847359s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
2145 | 2024-05-15T07:12:17.202Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 613.853296ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
2146 | 2024-05-15T07:12:17.228Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3") failed: Property not found. retry in 1.164717044s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
2147 | 2024-05-15T07:12:17.379Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 1.442480685s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
2148 | 2024-05-15T07:12:17.413Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 584.773164ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
2149 | 2024-05-15T07:12:17.481Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 1.248568006s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2150 | 2024-05-15T07:12:17.499Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 629.774355ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
2151 | 2024-05-15T07:12:17.571Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 642.826381ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
2152 | 2024-05-15T07:12:17.628Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959") failed: Property not found. retry in 846.772815ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2153 | 2024-05-15T07:12:17.640Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 543.4799ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
2154 | 2024-05-15T07:12:17.780Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 1.391797863s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
2155 | 2024-05-15T07:12:17.809Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 771.575832ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
2156 | 2024-05-15T07:12:17.831Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 503.859158ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
2157 | 2024-05-15T07:12:18.010Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054") failed: Property not found. retry in 1.287753794s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
2158 | 2024-05-15T07:12:18.034Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05") failed: Property not found. retry in 1.189568454s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
|
2159 | 2024-05-15T07:12:18.146Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 675.214636ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
2160 | 2024-05-15T07:12:18.196Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e") failed: Property not found. retry in 873.506177ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
2161 | 2024-05-15T07:12:18.207Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 518.999038ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
2162 | 2024-05-15T07:12:18.228Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 502.924566ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
2163 | 2024-05-15T07:12:18.250Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8") failed: Property not found. retry in 761.042156ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
2164 | 2024-05-15T07:12:18.352Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 1.487241809s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
2165 | 2024-05-15T07:12:18.406Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3") failed: Property not found. retry in 1.449707724s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
2166 | 2024-05-15T07:12:18.492Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959") failed: Property not found. retry in 1.279970867s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2167 | 2024-05-15T07:12:18.595Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b") failed: Property not found. retry in 741.054291ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
2168 | 2024-05-15T07:12:18.743Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 979.168261ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
2169 | 2024-05-15T07:12:18.761Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 1.218236171s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2170 | 2024-05-15T07:12:18.775Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ecac0828-7786-4503-af9f-835e69870288") failed: Property not found. retry in 995.316115ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
2171 | 2024-05-15T07:12:18.837Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc") failed: Property not found. retry in 697.991814ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
2172 | 2024-05-15T07:12:18.850Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550") failed: Property not found. retry in 1.24416045s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
2173 | 2024-05-15T07:12:19.189Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 779.848844ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
2174 | 2024-05-15T07:12:19.242Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05") failed: Property not found. retry in 1.006240209s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
|
2175 | 2024-05-15T07:12:19.739Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 723.503383ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
2176 | 2024-05-15T07:12:19.807Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959") failed: Property not found. retry in 1.446870095s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2177 | 2024-05-15T07:12:19.852Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 504.748212ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
2178 | 2024-05-15T07:12:19.868Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3") failed: Property not found. retry in 747.188043ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
2179 | 2024-05-15T07:12:19.981Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 1.092358054s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
2180 | 2024-05-15T07:12:19.993Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 718.369455ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2181 | 2024-05-15T07:12:20.260Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05") failed: Property not found. retry in 1.399675741s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
|
2182 | 2024-05-15T07:12:20.368Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 872.868086ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
2183 | 2024-05-15T07:12:20.473Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e") failed: Property not found. retry in 1.319788345s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
2184 | 2024-05-15T07:12:20.631Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3") failed: Property not found. retry in 1.225429317s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
2185 | 2024-05-15T07:12:20.725Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f") failed: Property not found. retry in 1.141244141s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2186 | 2024-05-15T07:12:21.086Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834") failed: Property not found. retry in 1.306120121s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
2187 | 2024-05-15T07:12:21.252Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25") failed: Property not found. retry in 536.034198ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
2188 | 2024-05-15T07:12:21.264Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959") failed: Property not found. retry in 817.142215ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2189 | 2024-05-15T07:12:21.678Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05") failed: Property not found. retry in 1.088846718s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
|
2190 | 2024-05-15T07:12:22.811Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834
|
2191 | 2024-05-15T07:12:22.811Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25
|
2192 | 2024-05-15T07:12:22.811Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e
|
2193 | 2024-05-15T07:12:22.811Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8
|
2194 | 2024-05-15T07:12:22.811Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b
|
2195 | 2024-05-15T07:12:22.811Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607
|
2196 | 2024-05-15T07:12:22.811Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e
|
2197 | 2024-05-15T07:12:22.811Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f
|
2198 | 2024-05-15T07:12:22.811Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959
|
2199 | 2024-05-15T07:12:22.811Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc
|
2200 | 2024-05-15T07:12:22.811Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d
|
2201 | 2024-05-15T07:12:22.811Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054
|
2202 | 2024-05-15T07:12:22.813Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b
|
2203 | 2024-05-15T07:12:22.813Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8
|
2204 | 2024-05-15T07:12:22.813Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a
|
2205 | 2024-05-15T07:12:22.813Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_ecac0828-7786-4503-af9f-835e69870288
|
2206 | 2024-05-15T07:12:22.813Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550
|
2207 | 2024-05-15T07:12:22.814Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05
|
2208 | 2024-05-15T07:12:22.814Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3026
zone = oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3
|
2209 | 2024-05-15T07:12:22.823Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
2210 | 2024-05-15T07:12:22.825Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
2211 | 2024-05-15T07:12:22.834Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 24698311
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:60075
req_id = c072f072-967b-4315-9f5d-017448974204
response_code = 204
uri = /omicron-zones
|
2212 | 2024-05-15T07:12:22.867Z | INFO | SledAgent (RSS): Finished setting up services
file = sled-agent/src/rack_setup/service.rs:1236
|
2213 | 2024-05-15T07:12:22.868Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.rss-plan-completed.marker.tmp
file = common/src/ledger.rs:199
|
2214 | 2024-05-15T07:12:22.868Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.rss-plan-completed.marker.tmp
file = common/src/ledger.rs:199
|
2215 | 2024-05-15T07:12:22.881Z | INFO | SledAgent (RSS): Handing off control to Nexus
file = sled-agent/src/rack_setup/service.rs:680
|
2216 | 2024-05-15T07:12:22.894Z | INFO | SledAgent (RSS): Nexus address: [fd00:1122:3344:101::a]:12221
file = sled-agent/src/rack_setup/service.rs:693
|
2217 | 2024-05-15T07:12:22.974Z | INFO | SledAgent (RSS): rack_network_config: RackNetworkConfigV1 {
bfd: [],
bgp: [],
infra_ip_first: 10.151.2.174,
infra_ip_last: 10.151.2.174,
ports: [
PortConfigV1 {
addresses: [
V4(
Ipv4Network {
addr: 10.151.2.174,
prefix: 24,
},
),
],
autoneg: false,
bgp_peers: [],
port: "qsfp0",
routes: [
RouteConfig {
destination: V4(
Ipv4Network {
addr: 0.0.0.0,
prefix: 0,
},
),
nexthop: 10.151.2.100,
vlan_id: None,
},
],
switch: Switch0,
uplink_port_fec: None,
uplink_port_speed: Speed40G,
},
],
rack_subnet: Ipv6Network {
addr: fd00:1122:3344:100::,
prefix: 56,
},
}
file = sled-agent/src/rack_setup/service.rs:817
|
2218 | 2024-05-15T07:12:23.307Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/nexus.rs:619
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
2219 | 2024-05-15T07:12:23.314Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2220 | 2024-05-15T07:12:23.464Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2221 | 2024-05-15T07:12:23.550Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2222 | 2024-05-15T07:12:23.627Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2223 | 2024-05-15T07:12:23.807Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2224 | 2024-05-15T07:12:23.916Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2225 | 2024-05-15T07:12:24.192Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2226 | 2024-05-15T07:12:24.496Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2227 | 2024-05-15T07:12:24.969Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2228 | 2024-05-15T07:12:25.075Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/nexus.rs:619
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
2229 | 2024-05-15T07:12:25.188Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2230 | 2024-05-15T07:12:25.560Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2231 | 2024-05-15T07:12:25.942Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2232 | 2024-05-15T07:12:26.991Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2233 | 2024-05-15T07:12:27.075Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/nexus.rs:619
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
2234 | 2024-05-15T07:12:28.106Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2235 | 2024-05-15T07:12:29.070Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/nexus.rs:619
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
2236 | 2024-05-15T07:12:29.095Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/probe_manager.rs:169
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
2237 | 2024-05-15T07:12:29.438Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2238 | 2024-05-15T07:12:30.098Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/probe_manager.rs:169
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
2239 | 2024-05-15T07:12:30.878Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:879
|
2240 | 2024-05-15T07:12:31.066Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/nexus.rs:619
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
2241 | 2024-05-15T07:12:31.101Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/probe_manager.rs:169
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
2242 | 2024-05-15T07:12:32.104Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/probe_manager.rs:169
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
2243 | 2024-05-15T07:12:32.894Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 403 Forbidden; headers: {"content-type": "application/json", "x-request-id": "3d298a7a-a40d-470b-b20b-940db826d743", "content-length": "113", "date": "Wed, 15 May 2024 07:12:32 GMT"}; value: Error { error_code: Some("Forbidden"), message: "Forbidden", request_id: "3d298a7a-a40d-470b-b20b-940db826d743" }
file = sled-agent/src/rack_setup/service.rs:879
|
2244 | 2024-05-15T07:12:33.526Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:59637
|
2245 | 2024-05-15T07:12:33.527Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 237
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:59637
req_id = 5c42073e-5377-4592-9def-5e72c6986bd7
response_code = 200
uri = /baseboard
|
2246 | 2024-05-15T07:12:34.987Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "b79cbd22-932d-4a9d-a4c4-c77c63bd79f5", "content-length": "173", "date": "Wed, 15 May 2024 07:12:34 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"acb2a796-8e9f-4811-b601-726a12d79b0b\"", request_id: "b79cbd22-932d-4a9d-a4c4-c77c63bd79f5" }
file = sled-agent/src/rack_setup/service.rs:879
|
2247 | 2024-05-15T07:12:35.131Z | INFO | SledAgent (NexusNotifierTask): Successfully put SledAgentInfo to nexus
file = sled-agent/src/nexus.rs:592
new_generation = 1
old_generation = 1
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
2248 | 2024-05-15T07:12:35.257Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:37863
|
2249 | 2024-05-15T07:12:35.259Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
2250 | 2024-05-15T07:12:35.352Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 94159
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:37863
req_id = 8bb9e1e4-828b-480f-8dde-3edab2b98673
response_code = 200
uri = /inventory
|
2251 | 2024-05-15T07:12:35.354Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
2252 | 2024-05-15T07:12:35.354Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
2253 | 2024-05-15T07:12:35.356Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
2254 | 2024-05-15T07:12:35.356Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 4215
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:37863
req_id = 205076cb-ce30-4ac6-8e07-166edaceb5a8
response_code = 200
uri = /omicron-zones
|
2255 | 2024-05-15T07:12:38.779Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:61404
|
2256 | 2024-05-15T07:12:38.782Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name("external-dns-e5538333-fcad-480c-9181-aa7e94aaf834"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name("external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name("nexus-99861af6-438d-4bef-ba93-632645b46d2f"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name("nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name("nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
2257 | 2024-05-15T07:12:38.782Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
2258 | 2024-05-15T07:12:38.782Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2259 | 2024-05-15T07:12:38.782Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2260 | 2024-05-15T07:12:38.782Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2261 | 2024-05-15T07:12:38.782Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
2262 | 2024-05-15T07:12:38.782Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
2263 | 2024-05-15T07:12:38.782Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 3180
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:61404
req_id = 02e530e1-39fc-4701-a212-3b271b8e3b73
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
2264 | 2024-05-15T07:12:38.785Z | INFO | SledAgent (RSS): Handoff to Nexus is complete
file = sled-agent/src/rack_setup/service.rs:889
|
2265 | 2024-05-15T07:12:38.934Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:57963
|
2266 | 2024-05-15T07:12:38.934Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
2267 | 2024-05-15T07:12:39.041Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 107667
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:57963
req_id = 9d8dc2ca-ff05-41df-a8e2-ba3fe70befbb
response_code = 200
uri = /inventory
|
2268 | 2024-05-15T07:12:39.042Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:43216
|
2269 | 2024-05-15T07:12:39.042Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
2270 | 2024-05-15T07:12:39.042Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
2271 | 2024-05-15T07:12:39.042Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name("external-dns-e5538333-fcad-480c-9181-aa7e94aaf834"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name("external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name("nexus-99861af6-438d-4bef-ba93-632645b46d2f"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name("nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name("nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
2272 | 2024-05-15T07:12:39.042Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
2273 | 2024-05-15T07:12:39.042Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2274 | 2024-05-15T07:12:39.042Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2275 | 2024-05-15T07:12:39.042Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2276 | 2024-05-15T07:12:39.042Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
2277 | 2024-05-15T07:12:39.042Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
2278 | 2024-05-15T07:12:39.042Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 592
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:43216
req_id = 9ad20360-890f-488d-ae51-0f889648f461
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
2279 | 2024-05-15T07:12:39.042Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json
file = common/src/ledger.rs:177
|
2280 | 2024-05-15T07:12:39.043Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 1009
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:57963
req_id = 1a0a8a9d-6087-4119-9f1a-9929afb0490f
response_code = 200
uri = /omicron-zones
|
2281 | 2024-05-15T07:12:39.062Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:36449
|
2282 | 2024-05-15T07:12:39.063Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name("external-dns-e5538333-fcad-480c-9181-aa7e94aaf834"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name("external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name("nexus-99861af6-438d-4bef-ba93-632645b46d2f"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name("nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name("nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
2283 | 2024-05-15T07:12:39.063Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
2284 | 2024-05-15T07:12:39.063Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2285 | 2024-05-15T07:12:39.063Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2286 | 2024-05-15T07:12:39.063Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2287 | 2024-05-15T07:12:39.063Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
2288 | 2024-05-15T07:12:39.063Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
2289 | 2024-05-15T07:12:39.063Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 899
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:36449
req_id = ba17ed3d-9f39-445c-830a-acdde3c7bbf1
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
2290 | 2024-05-15T07:12:39.119Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:35769
|
2291 | 2024-05-15T07:12:39.119Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name("external-dns-e5538333-fcad-480c-9181-aa7e94aaf834"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name("external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name("nexus-99861af6-438d-4bef-ba93-632645b46d2f"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name("nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name("nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
2292 | 2024-05-15T07:12:39.119Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
2293 | 2024-05-15T07:12:39.119Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2294 | 2024-05-15T07:12:39.119Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2295 | 2024-05-15T07:12:39.119Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2296 | 2024-05-15T07:12:39.119Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
2297 | 2024-05-15T07:12:39.119Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
2298 | 2024-05-15T07:12:39.119Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 317
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:35769
req_id = 3d7e6a9a-1e6a-41a5-a4f0-fb0ddb8a65c7
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
2299 | 2024-05-15T07:13:02.948Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:52531
|
2300 | 2024-05-15T07:13:03.196Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 248208
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:52531
req_id = 28bc9a69-71bd-4fc2-8bac-5108a2ff243b
response_code = 204
uri = /switch-ports
|
2301 | 2024-05-15T07:13:03.197Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:49544
|
2302 | 2024-05-15T07:13:03.197Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:43840
|
2303 | 2024-05-15T07:13:03.207Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 10586
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:52531
req_id = 9f4826e5-c9ca-4732-ae85-483f78b1ea79
response_code = 200
uri = /network-bootstore-config
|
2304 | 2024-05-15T07:13:03.264Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 67436
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:43840
req_id = e0aee942-eb3e-439a-880e-b79223f15578
response_code = 204
uri = /switch-ports
|
2305 | 2024-05-15T07:13:03.268Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 667
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:43840
req_id = 8299ffd6-6b63-4d3c-9639-696ca9d6d066
response_code = 200
uri = /network-bootstore-config
|
2306 | 2024-05-15T07:13:03.302Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 2, current_generation: 1
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
2307 | 2024-05-15T07:13:03.310Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
2308 | 2024-05-15T07:13:03.315Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
2309 | 2024-05-15T07:13:03.316Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 2
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-sock-i86pc
|
2310 | 2024-05-15T07:13:03.316Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 3, current_generation: 2
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
2311 | 2024-05-15T07:13:03.316Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
2312 | 2024-05-15T07:13:03.316Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 26688
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:52531
req_id = 7b200b6c-15f2-4e13-b9f5-9b9bf79a627d
response_code = 204
uri = /network-bootstore-config
|
2313 | 2024-05-15T07:13:03.316Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
2314 | 2024-05-15T07:13:03.316Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 3
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-sock-i86pc
|
2315 | 2024-05-15T07:13:03.316Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 13480
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:43840
req_id = f437cb59-2aa6-433a-9663-0f1b24219639
response_code = 204
uri = /network-bootstore-config
|
2316 | 2024-05-15T07:13:03.323Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 126368
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:49544
req_id = ef35cae4-c56a-49ee-92bf-a967c27388a5
response_code = 204
uri = /switch-ports
|
2317 | 2024-05-15T07:13:03.323Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 102
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:49544
req_id = 05a1649b-612e-4a55-b5a8-8f7d34ece348
response_code = 200
uri = /network-bootstore-config
|
2318 | 2024-05-15T07:13:03.560Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45410
|
2319 | 2024-05-15T07:13:03.560Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 36
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45410
req_id = 233b3317-ec58-479a-869f-a47c4c40cae9
response_code = 200
uri = /baseboard
|
2320 | 2024-05-15T07:13:20.838Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
2321 | 2024-05-15T07:13:33.135Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:44762
|
2322 | 2024-05-15T07:13:33.185Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 50064
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:44762
req_id = 398cfca2-28fc-4e98-a923-0c771c19a7cb
response_code = 204
uri = /switch-ports
|
2323 | 2024-05-15T07:13:33.185Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:35727
|
2324 | 2024-05-15T07:13:33.189Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 3185
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:44762
req_id = f865349d-bf7a-4c34-ab14-2300ce7b3dce
response_code = 200
uri = /network-bootstore-config
|
2325 | 2024-05-15T07:13:33.202Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:55652
|
2326 | 2024-05-15T07:13:33.235Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 49954
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:35727
req_id = 8ca03e7a-ac82-483d-a49b-ed393364ed78
response_code = 204
uri = /switch-ports
|
2327 | 2024-05-15T07:13:33.239Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 986
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:35727
req_id = 00fc9f5f-9aa9-45e7-85c5-8fe7afc30c4f
response_code = 200
uri = /network-bootstore-config
|
2328 | 2024-05-15T07:13:33.282Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 79262
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:55652
req_id = 7b7de0ae-6d0c-4501-a248-62740839f63f
response_code = 204
uri = /switch-ports
|
2329 | 2024-05-15T07:13:33.282Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 127
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:55652
req_id = 4a221d60-f3ec-4e3a-9fd9-8a6de2c39908
response_code = 200
uri = /network-bootstore-config
|
2330 | 2024-05-15T07:13:33.594Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:34286
|
2331 | 2024-05-15T07:13:33.594Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 105
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:34286
req_id = b4981772-3d6f-49c7-b1d8-9bf0a075ec30
response_code = 200
uri = /baseboard
|
2332 | 2024-05-15T07:14:02.909Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:60996
|
2333 | 2024-05-15T07:14:02.959Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 49424
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:60996
req_id = beddff37-a382-4d61-904f-aba722256fe0
response_code = 204
uri = /switch-ports
|
2334 | 2024-05-15T07:14:02.959Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 210
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:60996
req_id = 3f578d53-8574-46fa-9aa8-d03c564af5ea
response_code = 200
uri = /network-bootstore-config
|
2335 | 2024-05-15T07:14:02.968Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:35320
|
2336 | 2024-05-15T07:14:03.015Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 47309
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:35320
req_id = da7002e7-726e-4a6f-9c72-d82f4f76d353
response_code = 204
uri = /switch-ports
|
2337 | 2024-05-15T07:14:03.015Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:64339
|
2338 | 2024-05-15T07:14:03.019Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 3709
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:35320
req_id = b614e5d8-1a22-48a1-8c33-713d98f1ffed
response_code = 200
uri = /network-bootstore-config
|
2339 | 2024-05-15T07:14:03.062Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 46405
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:64339
req_id = daff3b85-32b6-4627-ac8b-a61a917fee18
response_code = 204
uri = /switch-ports
|
2340 | 2024-05-15T07:14:03.062Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 167
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:64339
req_id = b1045de8-73c2-48e8-947a-5e584d0505b4
response_code = 200
uri = /network-bootstore-config
|
2341 | 2024-05-15T07:14:03.628Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:44913
|
2342 | 2024-05-15T07:14:03.629Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 748
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:44913
req_id = 04b63b70-1d66-42ed-8a18-0b6c424f9eed
response_code = 200
uri = /baseboard
|
2343 | 2024-05-15T07:14:10.263Z | INFO | SledAgent (dropshot): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:55688
remote_addr = [fd00:1122:3344:101::d]:65149
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
|
2344 | 2024-05-15T07:14:10.271Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 6994
local_addr = [fd00:1122:3344:101::1]:55688
method = GET
remote_addr = [fd00:1122:3344:101::d]:65149
req_id = 05e61014-d288-48f3-abc1-40686d810572
response_code = 200
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
uri = //acb2a796-8e9f-4811-b601-726a12d79b0b
|
2345 | 2024-05-15T07:14:33.179Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:63438
|
2346 | 2024-05-15T07:14:33.228Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 49360
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:63438
req_id = b2332f70-30d8-4049-9ff6-a90999c57f69
response_code = 204
uri = /switch-ports
|
2347 | 2024-05-15T07:14:33.228Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 100
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:63438
req_id = b6a6d308-ebad-40ab-b786-febd7cd991e2
response_code = 200
uri = /network-bootstore-config
|
2348 | 2024-05-15T07:14:33.310Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:52997
|
2349 | 2024-05-15T07:14:33.359Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 49443
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:52997
req_id = 1028b17c-24b6-4f71-8afb-957a02735929
response_code = 204
uri = /switch-ports
|
2350 | 2024-05-15T07:14:33.359Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:35614
|
2351 | 2024-05-15T07:14:33.360Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 213
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:52997
req_id = 5efed80f-370e-4a01-b84c-10560607c20a
response_code = 200
uri = /network-bootstore-config
|
2352 | 2024-05-15T07:14:33.407Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 47495
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:35614
req_id = c132f461-b434-4b96-b0ca-c93b8869a117
response_code = 204
uri = /switch-ports
|
2353 | 2024-05-15T07:14:33.407Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 147
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:35614
req_id = 59e9806c-713d-4ed5-b8b7-57552c509996
response_code = 200
uri = /network-bootstore-config
|
2354 | 2024-05-15T07:14:33.660Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:48814
|
2355 | 2024-05-15T07:14:33.661Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 89
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:48814
req_id = 1fc36acd-4f1d-4592-93c1-03aa46db3acb
response_code = 200
uri = /baseboard
|
2356 | 2024-05-15T07:14:40.262Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 1233
local_addr = [fd00:1122:3344:101::1]:55688
method = GET
remote_addr = [fd00:1122:3344:101::d]:65149
req_id = 6c4a0b17-7b19-4008-8484-8da5257a493b
response_code = 200
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
uri = //acb2a796-8e9f-4811-b601-726a12d79b0b
|
2357 | 2024-05-15T07:15:02.981Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:60589
|
2358 | 2024-05-15T07:15:03.041Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 59481
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:60589
req_id = 36449bb0-3ac6-4f3e-aa44-54a7b930ca8e
response_code = 204
uri = /switch-ports
|
2359 | 2024-05-15T07:15:03.041Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:35240
|
2360 | 2024-05-15T07:15:03.089Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 48105
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:35240
req_id = 80c893df-17ee-488d-bd0e-5dfc8ff37d63
response_code = 204
uri = /switch-ports
|
2361 | 2024-05-15T07:15:03.089Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 107
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:60589
req_id = 0f9b5950-51f3-4004-a26a-75fe609dbf60
response_code = 200
uri = /network-bootstore-config
|
2362 | 2024-05-15T07:15:03.090Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 791
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:35240
req_id = 6d67ada2-1e64-45ad-9edd-9f5da103713d
response_code = 200
uri = /network-bootstore-config
|
2363 | 2024-05-15T07:15:03.107Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:62753
|
2364 | 2024-05-15T07:15:03.156Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 48741
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:62753
req_id = 964f7f9d-0b46-4039-a788-9acb3ffd61c5
response_code = 204
uri = /switch-ports
|
2365 | 2024-05-15T07:15:03.156Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 108
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:62753
req_id = 7f82a5fe-dd7a-4401-8325-2324b157352e
response_code = 200
uri = /network-bootstore-config
|
2366 | 2024-05-15T07:15:03.694Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:51655
|
2367 | 2024-05-15T07:15:03.694Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 78
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:51655
req_id = 12776a45-b514-45db-a893-f7b27ff570ef
response_code = 200
uri = /baseboard
|
2368 | 2024-05-15T07:15:10.262Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 775
local_addr = [fd00:1122:3344:101::1]:55688
method = GET
remote_addr = [fd00:1122:3344:101::d]:65149
req_id = dec04add-b9ac-4431-ab87-3dd1cfb7340e
response_code = 200
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
uri = //acb2a796-8e9f-4811-b601-726a12d79b0b
|
2369 | 2024-05-15T07:15:33.101Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:54570
|
2370 | 2024-05-15T07:15:33.152Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 51104
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:54570
req_id = 3bc4c5cb-489e-4fb5-9c7a-c6ab1063fc22
response_code = 204
uri = /switch-ports
|
2371 | 2024-05-15T07:15:33.153Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:33472
|
2372 | 2024-05-15T07:15:33.156Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 3361
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:54570
req_id = aa61d6bc-2b3e-4d02-9842-4a2a94bf2b25
response_code = 200
uri = /network-bootstore-config
|
2373 | 2024-05-15T07:15:33.199Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 46362
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:33472
req_id = 8a27fe0f-1913-4d4a-b951-c1a1e59e9544
response_code = 204
uri = /switch-ports
|
2374 | 2024-05-15T07:15:33.200Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 1108
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:33472
req_id = d3e9e866-417a-41ae-9ace-e013b1e270b5
response_code = 200
uri = /network-bootstore-config
|
2375 | 2024-05-15T07:15:33.243Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:58717
|
2376 | 2024-05-15T07:15:33.290Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 46535
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:58717
req_id = c99d36af-89c0-46b0-8dcc-ec616b368cfa
response_code = 204
uri = /switch-ports
|
2377 | 2024-05-15T07:15:33.290Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 97
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:58717
req_id = 093c0b29-54eb-4157-954a-72e6f664193c
response_code = 200
uri = /network-bootstore-config
|
2378 | 2024-05-15T07:15:33.728Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:37563
|
2379 | 2024-05-15T07:15:33.728Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 77
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:37563
req_id = 8ad307c4-0365-4827-b8af-9311d6d0b03b
response_code = 200
uri = /baseboard
|
2380 | 2024-05-15T07:15:40.261Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 591
local_addr = [fd00:1122:3344:101::1]:55688
method = GET
remote_addr = [fd00:1122:3344:101::d]:65149
req_id = 399eac94-8b08-466d-9316-7c24e014a305
response_code = 200
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
uri = //acb2a796-8e9f-4811-b601-726a12d79b0b
|
2381 | 2024-05-15T07:16:02.908Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:34134
|
2382 | 2024-05-15T07:16:02.958Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 49974
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:34134
req_id = b88c276e-68d2-4285-9b07-3f559b8d14dc
response_code = 204
uri = /switch-ports
|
2383 | 2024-05-15T07:16:02.958Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 102
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:34134
req_id = da56b1a8-3983-4363-b7c1-b59de8d17585
response_code = 200
uri = /network-bootstore-config
|
2384 | 2024-05-15T07:16:02.984Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:42671
|
2385 | 2024-05-15T07:16:03.031Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 46892
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:42671
req_id = dcd2c5fb-0918-4af0-88b2-860a9228f29c
response_code = 204
uri = /switch-ports
|
2386 | 2024-05-15T07:16:03.031Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 103
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:42671
req_id = b27630f0-7534-4c06-b11d-c6ead4a898c1
response_code = 200
uri = /network-bootstore-config
|
2387 | 2024-05-15T07:16:03.075Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:56470
|
2388 | 2024-05-15T07:16:03.122Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 46565
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:56470
req_id = ea07257b-e0ce-4abe-a469-6c9d36ca3665
response_code = 204
uri = /switch-ports
|
2389 | 2024-05-15T07:16:03.122Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 97
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:56470
req_id = 95ca3787-0fc2-495c-a824-675f651e3d4e
response_code = 200
uri = /network-bootstore-config
|
2390 | 2024-05-15T07:16:03.769Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:61240
|
2391 | 2024-05-15T07:16:03.769Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 99
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:61240
req_id = 6ef44f29-34d2-4950-80f5-2b577cc1aa84
response_code = 200
uri = /baseboard
|
2392 | 2024-05-15T07:16:10.274Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 884
local_addr = [fd00:1122:3344:101::1]:55688
method = GET
remote_addr = [fd00:1122:3344:101::d]:65149
req_id = 03d350f2-1166-4f34-a506-659ce5186e17
response_code = 200
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
uri = //acb2a796-8e9f-4811-b601-726a12d79b0b
|
2393 | 2024-05-15T07:16:33.118Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:59864
|
2394 | 2024-05-15T07:16:33.168Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 49414
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:59864
req_id = ef700789-750d-4890-909d-faec180e2044
response_code = 204
uri = /switch-ports
|
2395 | 2024-05-15T07:16:33.168Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 97
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:59864
req_id = 75d82a7e-a12b-4d4a-b95e-11dd8637b425
response_code = 200
uri = /network-bootstore-config
|
2396 | 2024-05-15T07:16:33.175Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:37882
|
2397 | 2024-05-15T07:16:33.225Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 49116
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:37882
req_id = ceeff066-b412-41de-9722-a6d6bc38e5b2
response_code = 204
uri = /switch-ports
|
2398 | 2024-05-15T07:16:33.225Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:44747
|
2399 | 2024-05-15T07:16:33.228Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 3334
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:37882
req_id = 3b130e9e-97e5-4cbb-8762-514d34343a0c
response_code = 200
uri = /network-bootstore-config
|
2400 | 2024-05-15T07:16:33.271Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 46224
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:44747
req_id = 8f77a1dd-6d51-4a50-b3ad-73f2001f9d06
response_code = 204
uri = /switch-ports
|
2401 | 2024-05-15T07:16:33.271Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 137
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:44747
req_id = 82ec753d-08ba-49a6-8f2d-c5930aef8540
response_code = 200
uri = /network-bootstore-config
|
2402 | 2024-05-15T07:16:33.801Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:47426
|
2403 | 2024-05-15T07:16:33.801Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 69
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:47426
req_id = e26513a4-0a72-4b78-9897-bcfd91e5fb3f
response_code = 200
uri = /baseboard
|
2404 | 2024-05-15T07:16:40.310Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 735
local_addr = [fd00:1122:3344:101::1]:55688
method = GET
remote_addr = [fd00:1122:3344:101::d]:65149
req_id = 33088b5c-34c2-4641-b528-cb827b6fe05f
response_code = 200
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
uri = //acb2a796-8e9f-4811-b601-726a12d79b0b
|
2405 | 2024-05-15T07:17:02.915Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:36935
|
2406 | 2024-05-15T07:17:02.966Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 50315
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:36935
req_id = 5bca168d-11fb-4e93-bc06-e656ea964358
response_code = 204
uri = /switch-ports
|
2407 | 2024-05-15T07:17:02.966Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:56807
|
2408 | 2024-05-15T07:17:02.970Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 3963
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:36935
req_id = 994624f3-5f7e-4674-bdb7-94bd04721d04
response_code = 200
uri = /network-bootstore-config
|
2409 | 2024-05-15T07:17:03.015Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 49240
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:56807
req_id = addcbcc2-6626-4519-a575-0103b6112dbf
response_code = 204
uri = /switch-ports
|
2410 | 2024-05-15T07:17:03.015Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 133
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:56807
req_id = 156b7690-1207-4547-b988-2b559a85a53a
response_code = 200
uri = /network-bootstore-config
|
2411 | 2024-05-15T07:17:03.089Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:33083
|
2412 | 2024-05-15T07:17:03.134Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 44669
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:33083
req_id = c8eaabf7-6951-4f6b-b89b-1083afd1af9f
response_code = 204
uri = /switch-ports
|
2413 | 2024-05-15T07:17:03.134Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 106
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:33083
req_id = 83221fb5-99b6-4fc0-ad1f-75385bd81229
response_code = 200
uri = /network-bootstore-config
|
2414 | 2024-05-15T07:17:03.832Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:44995
|
2415 | 2024-05-15T07:17:03.832Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 89
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:44995
req_id = 894d0526-cf1f-4676-8933-ad4c04fffc57
response_code = 200
uri = /baseboard
|
2416 | 2024-05-15T07:17:10.310Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 620
local_addr = [fd00:1122:3344:101::1]:55688
method = GET
remote_addr = [fd00:1122:3344:101::d]:65149
req_id = a97ae30f-2b33-4f80-be98-ef985a0304bc
response_code = 200
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
uri = //acb2a796-8e9f-4811-b601-726a12d79b0b
|
2417 | 2024-05-15T07:17:33.045Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:45132
|
2418 | 2024-05-15T07:17:33.045Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name("external-dns-e5538333-fcad-480c-9181-aa7e94aaf834"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name("external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name("nexus-99861af6-438d-4bef-ba93-632645b46d2f"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name("nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name("nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
2419 | 2024-05-15T07:17:33.045Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
2420 | 2024-05-15T07:17:33.045Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2421 | 2024-05-15T07:17:33.045Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2422 | 2024-05-15T07:17:33.046Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2423 | 2024-05-15T07:17:33.046Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
2424 | 2024-05-15T07:17:33.046Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
2425 | 2024-05-15T07:17:33.046Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 377
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:45132
req_id = c116ff48-a53b-4574-80ca-116b0fb617e1
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
2426 | 2024-05-15T07:17:33.101Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:60191
|
2427 | 2024-05-15T07:17:33.149Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 48081
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:60191
req_id = d5e06d5a-dbc6-4193-be79-ddecfb4b0aef
response_code = 204
uri = /switch-ports
|
2428 | 2024-05-15T07:17:33.150Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:56776
|
2429 | 2024-05-15T07:17:33.150Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 187
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:60191
req_id = 0d55b02d-f9d3-4c1e-91c3-2ccd1f44e1f1
response_code = 200
uri = /network-bootstore-config
|
2430 | 2024-05-15T07:17:33.150Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name("external-dns-e5538333-fcad-480c-9181-aa7e94aaf834"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name("external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name("nexus-99861af6-438d-4bef-ba93-632645b46d2f"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name("nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name("nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
2431 | 2024-05-15T07:17:33.150Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
2432 | 2024-05-15T07:17:33.150Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2433 | 2024-05-15T07:17:33.150Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2434 | 2024-05-15T07:17:33.150Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2435 | 2024-05-15T07:17:33.150Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
2436 | 2024-05-15T07:17:33.150Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
2437 | 2024-05-15T07:17:33.150Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 496
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:56776
req_id = a88ed11f-061b-4ebc-aac1-8ebcbb4ecca3
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
2438 | 2024-05-15T07:17:33.169Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:34152
|
2439 | 2024-05-15T07:17:33.169Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name("external-dns-e5538333-fcad-480c-9181-aa7e94aaf834"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name("external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name("nexus-99861af6-438d-4bef-ba93-632645b46d2f"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name("nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name("nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
2440 | 2024-05-15T07:17:33.169Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
2441 | 2024-05-15T07:17:33.170Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2442 | 2024-05-15T07:17:33.170Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2443 | 2024-05-15T07:17:33.170Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
2444 | 2024-05-15T07:17:33.170Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
2445 | 2024-05-15T07:17:33.170Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
2446 | 2024-05-15T07:17:33.170Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 406
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:34152
req_id = 59663957-072b-4778-9039-c175fd8162c0
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
2447 | 2024-05-15T07:17:33.229Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:58296
|
2448 | 2024-05-15T07:17:33.279Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 49289
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:58296
req_id = 27187d12-4698-475e-8a1e-3b035f150a53
response_code = 204
uri = /switch-ports
|
2449 | 2024-05-15T07:17:33.279Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 117
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:58296
req_id = f8acf92b-29c1-473e-8659-7ccbbc752d1a
response_code = 200
uri = /network-bootstore-config
|
2450 | 2024-05-15T07:17:33.294Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:33458
|
2451 | 2024-05-15T07:17:33.339Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 45250
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:33458
req_id = 6df35e5d-75ff-4de8-84f3-0f54314c672e
response_code = 204
uri = /switch-ports
|
2452 | 2024-05-15T07:17:33.339Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 203
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:33458
req_id = 1d4481a5-ac6c-449c-aacf-2cb7263fb11d
response_code = 200
uri = /network-bootstore-config
|
2453 | 2024-05-15T07:17:33.865Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:64069
|
2454 | 2024-05-15T07:17:33.865Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 99
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:64069
req_id = bbca4da6-1609-44b9-8c76-a5bbc05b22a4
response_code = 200
uri = /baseboard
|
2455 | 2024-05-15T07:17:40.310Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 788
local_addr = [fd00:1122:3344:101::1]:55688
method = GET
remote_addr = [fd00:1122:3344:101::d]:65149
req_id = 5f1cf87e-c336-482a-8e99-c26151e53f60
response_code = 200
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
uri = //acb2a796-8e9f-4811-b601-726a12d79b0b
|
2456 | 2024-05-15T07:18:02.905Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:51220
|
2457 | 2024-05-15T07:18:02.955Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 49848
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:51220
req_id = bcc55ac7-fec5-496b-96aa-81fb815089a9
response_code = 204
uri = /switch-ports
|
2458 | 2024-05-15T07:18:02.955Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:55699
|
2459 | 2024-05-15T07:18:02.955Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:38710
|
2460 | 2024-05-15T07:18:02.961Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 5695
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:51220
req_id = 31d894db-2652-408b-97cd-ca3f2eec2ba4
response_code = 200
uri = /network-bootstore-config
|
2461 | 2024-05-15T07:18:03.004Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 48873
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:55699
req_id = 4f7f0d85-b9b5-49b9-85b8-fad07cc24570
response_code = 204
uri = /switch-ports
|
2462 | 2024-05-15T07:18:03.009Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 1341
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:55699
req_id = a3e8aab2-6689-4066-a1de-434396b22abe
response_code = 200
uri = /network-bootstore-config
|
2463 | 2024-05-15T07:18:03.051Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 94970
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:38710
req_id = 1ef7ee96-d501-4cd0-8955-ab386abbeddd
response_code = 204
uri = /switch-ports
|
2464 | 2024-05-15T07:18:03.051Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 112
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:38710
req_id = 729c1575-4c82-4072-949f-a5459e45c444
response_code = 200
uri = /network-bootstore-config
|
2465 | 2024-05-15T07:18:03.899Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:53753
|
2466 | 2024-05-15T07:18:03.899Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 62
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:53753
req_id = 3a98d705-2216-4d0c-95e9-839e0d56bd71
response_code = 200
uri = /baseboard
|
2467 | 2024-05-15T07:18:07.237Z | INFO | SledAgent (auto-cleanup-task): running automatic periodic zone bundle cleanup
file = sled-agent/src/zone_bundle.rs:300
|
2468 | 2024-05-15T07:18:07.238Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
2469 | 2024-05-15T07:18:07.276Z | INFO | SledAgent (auto-cleanup-task): top of bundle cleanup loop
file = sled-agent/src/zone_bundle.rs:289
next_cleanup = Instant { tv_sec: 1851, tv_nsec: 684877306 }
time_to_next_cleanup = 599.999997091s
|
2470 | 2024-05-15T07:18:10.309Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 456
local_addr = [fd00:1122:3344:101::1]:55688
method = GET
remote_addr = [fd00:1122:3344:101::d]:65149
req_id = b92bca1d-92c7-4f12-b3c8-084def773135
response_code = 200
sled_id = acb2a796-8e9f-4811-b601-726a12d79b0b
uri = //acb2a796-8e9f-4811-b601-726a12d79b0b
|
2471 | 2024-05-15T07:18:17.269Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:32773
|
2472 | 2024-05-15T07:18:17.279Z | INFO | SledAgent (InstanceManager): ensuring instance is registered
file = sled-agent/src/instance_manager.rs:546
hardware = InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname("localshark") }, nics: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(12006808))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: "nvme", name: "disk-7d72237e3cb", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: a2bdf7c7-d54c-4168-8d09-daf3c96c27c1, read_only_parent: Some(Volume { block_size: 512, id: 064e6874-7451-4713-8b9b-16ba9758e09b, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 98f450bf-ba2d-4a97-b47f-1d65c4edbb9e, key: Some("N/PJy4twLSmZ7nLWEfFW8oifHAzQ1XQV3m2X/kcGn88="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::13]:19001", "[fd00:1122:3344:101::12]:19001", "[fd00:1122:3344:101::19]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a2bdf7c7-d54c-4168-8d09-daf3c96c27c1, key: Some("08OcIC7oIXvo3LKwtBjCBiI0Wxg52LMrRfXsu93NGs8="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::16]:19001", "[fd00:1122:3344:101::19]:19002", "[fd00:1122:3344:101::13]:19002"] } }] } }], cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAB6SDqvWK9YAABIOq9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAe0g6r1ivWAAASDqvWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3NGI1ZDkzZS0yMjQxLTQzMTktODEyOS1mZTY3MTA4YzhmNDkiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTDlyZFRFMkR3WXU5WGloTWxwdDQ4ZzU3OUhjZkROMDQ0MXpKTVhIbUFUZSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") }
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
instance_runtime = InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }
metadata = InstanceMetadata { silo_id: 82ce2851-ee69-4933-b9a7-f0eee8dffe12, project_id: 22d38dba-1ffb-40fc-b3f3-4ee2a548bcf6 }
propolis_addr = [fd00:1122:3344:101::1:0]:12400
propolis_id = b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
vmm_runtime = VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-15T07:18:16.763717Z }
|
2473 | 2024-05-15T07:18:17.279Z | INFO | SledAgent (InstanceManager): registering new instance
file = sled-agent/src/instance_manager.rs:580
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2474 | 2024-05-15T07:18:17.279Z | INFO | SledAgent (InstanceManager): initializing new Instance
file = sled-agent/src/instance.rs:938
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
propolis_id = b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
state = InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname("localshark") }, nics: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(12006808))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: "nvme", name: "disk-7d72237e3cb", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: a2bdf7c7-d54c-4168-8d09-daf3c96c27c1, read_only_parent: Some(Volume { block_size: 512, id: 064e6874-7451-4713-8b9b-16ba9758e09b, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 98f450bf-ba2d-4a97-b47f-1d65c4edbb9e, key: Some("N/PJy4twLSmZ7nLWEfFW8oifHAzQ1XQV3m2X/kcGn88="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::13]:19001", "[fd00:1122:3344:101::12]:19001", "[fd00:1122:3344:101::19]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a2bdf7c7-d54c-4168-8d09-daf3c96c27c1, key: Some("08OcIC7oIXvo3LKwtBjCBiI0Wxg52LMrRfXsu93NGs8="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::16]:19001", "[fd00:1122:3344:101::19]:19002", "[fd00:1122:3344:101::13]:19002"] } }] } }], cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAB6SDqvWK9YAABIOq9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAe0g6r1ivWAAASDqvWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3NGI1ZDkzZS0yMjQxLTQzMTktODEyOS1mZTY3MTA4YzhmNDkiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTDlyZFRFMkR3WXU5WGloTWxwdDQ4ZzU3OUhjZkROMDQ0MXpKTVhIbUFUZSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-15T07:18:16.763717Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }
|
2475 | 2024-05-15T07:18:17.282Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 13118
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:32773
req_id = 1ab5419d-8210-446d-b805-9ef2d6bffd63
response_code = 200
uri = /instances/74b5d93e-2241-4319-8129-fe67108c8f49
|
2476 | 2024-05-15T07:18:17.397Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:55817
|
2477 | 2024-05-15T07:18:17.397Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 392
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:55817
req_id = a6cd5a60-9841-4bd2-95e5-e097ae602b03
response_code = 200
uri = /instances/74b5d93e-2241-4319-8129-fe67108c8f49/state
|
2478 | 2024-05-15T07:18:17.422Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte6", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 241, 86, 179]), slot: 0, vni: Vni { inner: 12006808 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: "vopte6" } }
|
2479 | 2024-05-15T07:18:17.423Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
2480 | 2024-05-15T07:18:17.459Z | INFO | SledAgent (InstanceManager): Configuring new Omicron zone: oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
file = illumos-utils/src/zone.rs:315
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2481 | 2024-05-15T07:18:17.480Z | INFO | SledAgent (InstanceManager): Installing Omicron zone: oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
file = illumos-utils/src/zone.rs:354
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2482 | 2024-05-15T07:18:18.856Z | INFO | SledAgent (InstanceManager): Profile for oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="system/illumos/propolis-server">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlInstance0"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::1:0"/>
<propval type="astring" name="listen_port" value="12400"/>
<propval type="astring" name="metric_addr" value="[fd00:1122:3344:101::c]:12221"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2483 | 2024-05-15T07:18:18.856Z | INFO | SledAgent (InstanceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
zone = oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2484 | 2024-05-15T07:18:19.017Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543") failed: Property not found. retry in 59.142966ms
file = illumos-utils/src/svc.rs:36
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
zone = oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2485 | 2024-05-15T07:18:19.092Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543") failed: Property not found. retry in 59.936349ms
file = illumos-utils/src/svc.rs:36
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
zone = oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2486 | 2024-05-15T07:18:19.169Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543") failed: Property not found. retry in 113.110852ms
file = illumos-utils/src/svc.rs:36
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
zone = oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2487 | 2024-05-15T07:18:19.300Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543") failed: Property not found. retry in 577.037868ms
file = illumos-utils/src/svc.rs:36
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
zone = oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2488 | 2024-05-15T07:18:19.888Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543") failed: Property not found. retry in 1.010831071s
file = illumos-utils/src/svc.rs:36
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
zone = oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2489 | 2024-05-15T07:18:20.837Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:831
|
2490 | 2024-05-15T07:18:20.909Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543") failed: Property not found. retry in 1.036408746s
file = illumos-utils/src/svc.rs:36
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
zone = oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2491 | 2024-05-15T07:18:21.958Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543") failed: Property not found. retry in 796.304506ms
file = illumos-utils/src/svc.rs:36
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
zone = oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2492 | 2024-05-15T07:18:22.767Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543") failed: Property not found. retry in 715.342924ms
file = illumos-utils/src/svc.rs:36
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
zone = oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2493 | 2024-05-15T07:18:23.493Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543") failed: Property not found. retry in 1.320581035s
file = illumos-utils/src/svc.rs:36
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
zone = oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2494 | 2024-05-15T07:18:24.841Z | INFO | SledAgent (InstanceManager): Started propolis in zone: oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
file = sled-agent/src/instance.rs:1423
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2495 | 2024-05-15T07:18:24.849Z | INFO | SledAgent (InstanceManager): Propolis SMF service is online
file = sled-agent/src/instance.rs:1432
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2496 | 2024-05-15T07:18:24.884Z | INFO | SledAgent (InstanceManager): Propolis HTTP server online
file = sled-agent/src/instance.rs:1446
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2497 | 2024-05-15T07:18:24.886Z | INFO | SledAgent (InstanceManager): Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAB6SDqvWK9YAABIOq9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAe0g6r1ivWAAASDqvWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3NGI1ZDkzZS0yMjQxLTQzMTktODEyOS1mZTY3MTA4YzhmNDkiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTDlyZFRFMkR3WXU5WGloTWxwdDQ4ZzU3OUhjZkROMDQ0MXpKTVhIbUFUZSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"), disks: [DiskRequest { device: "nvme", name: "disk-7d72237e3cb", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: a2bdf7c7-d54c-4168-8d09-daf3c96c27c1, read_only_parent: Some(Volume { block_size: 512, id: 064e6874-7451-4713-8b9b-16ba9758e09b, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 98f450bf-ba2d-4a97-b47f-1d65c4edbb9e, key: Some("N/PJy4twLSmZ7nLWEfFW8oifHAzQ1XQV3m2X/kcGn88="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::13]:19001", "[fd00:1122:3344:101::12]:19001", "[fd00:1122:3344:101::19]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a2bdf7c7-d54c-4168-8d09-daf3c96c27c1, key: Some("08OcIC7oIXvo3LKwtBjCBiI0Wxg52LMrRfXsu93NGs8="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::16]:19001", "[fd00:1122:3344:101::19]:19002", "[fd00:1122:3344:101::13]:19002"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: "vopte6", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: "Test description", id: 74b5d93e-2241-4319-8129-fe67108c8f49, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 22d38dba-1ffb-40fc-b3f3-4ee2a548bcf6, silo_id: 82ce2851-ee69-4933-b9a7-f0eee8dffe12 }, name: "localshark", vcpus: 2 } }
file = sled-agent/src/instance.rs:685
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2498 | 2024-05-15T07:18:25.067Z | INFO | SledAgent (InstanceManager): result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })
file = sled-agent/src/instance.rs:687
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2499 | 2024-05-15T07:18:25.069Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-05-15T07:18:25.069060454Z }
file = sled-agent/src/instance.rs:569
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2500 | 2024-05-15T07:18:25.069Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:591
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
new_instance_state = InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }
new_vmm_state = VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-15T07:18:25.069060454Z }
propolis_id = b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2501 | 2024-05-15T07:18:25.069Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }, propolis_id: b926d0d2-78b1-4d16-b1ed-d24aaeaf7543, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-15T07:18:25.069060454Z } }
|
2502 | 2024-05-15T07:18:25.127Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-05-15T07:18:25.127457469Z }
file = sled-agent/src/instance.rs:569
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2503 | 2024-05-15T07:18:25.127Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:591
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
new_instance_state = InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }
new_vmm_state = VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-15T07:18:25.127457469Z }
propolis_id = b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2504 | 2024-05-15T07:18:25.127Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }, propolis_id: b926d0d2-78b1-4d16-b1ed-d24aaeaf7543, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-15T07:18:25.127457469Z } }
|
2505 | 2024-05-15T07:18:32.933Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:50267
|
2506 | 2024-05-15T07:18:32.934Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 204
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:50267
req_id = 40203884-97a7-466f-980a-3136923c76ce
response_code = 200
uri = /instances/74b5d93e-2241-4319-8129-fe67108c8f49/state
|
2507 | 2024-05-15T07:18:33.028Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:64479
|
2508 | 2024-05-15T07:18:33.029Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 151
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:64479
req_id = ebc4ba71-5f07-419a-aebf-af82ff730189
response_code = 200
uri = /instances/74b5d93e-2241-4319-8129-fe67108c8f49/state
|
2509 | 2024-05-15T07:18:33.173Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:51655
|
2510 | 2024-05-15T07:18:33.173Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 72
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:51655
req_id = d8b25a24-acdc-42bf-8d59-e248dd96d12c
response_code = 200
uri = /instances/74b5d93e-2241-4319-8129-fe67108c8f49/state
|
2511 | 2024-05-15T07:18:33.175Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:36263
|
2512 | 2024-05-15T07:18:33.223Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 48465
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:36263
req_id = 963148da-eb4b-4365-be4c-4f5dc58fae0f
response_code = 204
uri = /switch-ports
|
2513 | 2024-05-15T07:18:33.224Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 638
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:36263
req_id = 7a5512d1-ee04-4237-9827-4104e81b6087
response_code = 200
uri = /network-bootstore-config
|
2514 | 2024-05-15T07:18:33.266Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:62458
|
2515 | 2024-05-15T07:18:33.314Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 47563
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:62458
req_id = f5eb8092-9b65-4afd-9394-6f6b5f0fe954
response_code = 204
uri = /switch-ports
|
2516 | 2024-05-15T07:18:33.314Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 98
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:62458
req_id = 2da0135a-afee-4331-beea-1f0e2e92ee83
response_code = 200
uri = /network-bootstore-config
|
2517 | 2024-05-15T07:18:33.323Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:42679
|
2518 | 2024-05-15T07:18:33.370Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 46702
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:42679
req_id = 1111ecf7-a186-4f17-a519-f0387065f518
response_code = 204
uri = /switch-ports
|
2519 | 2024-05-15T07:18:33.370Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 99
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:42679
req_id = 3ef5cddd-cc5d-4746-a887-3d8760409416
response_code = 200
uri = /network-bootstore-config
|
2520 | 2024-05-15T07:18:33.933Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:55095
|
2521 | 2024-05-15T07:18:33.933Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 71
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:55095
req_id = fb108a3e-8f3c-4e3d-aed5-7d261f118d40
response_code = 200
uri = /baseboard
|
2522 | 2024-05-15T07:18:34.845Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:59545
|
2523 | 2024-05-15T07:18:34.845Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 606
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:59545
req_id = b9df7b19-4860-482a-b149-1f31e8f09abd
response_code = 200
uri = /instances/74b5d93e-2241-4319-8129-fe67108c8f49/state
|
2524 | 2024-05-15T07:18:34.845Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-05-15T07:18:34.845824263Z }
file = sled-agent/src/instance.rs:569
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2525 | 2024-05-15T07:18:34.845Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:591
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
new_instance_state = InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }
new_vmm_state = VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-15T07:18:34.845824263Z }
propolis_id = b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2526 | 2024-05-15T07:18:34.845Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }, propolis_id: b926d0d2-78b1-4d16-b1ed-d24aaeaf7543, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-15T07:18:34.845824263Z } }
|
2527 | 2024-05-15T07:18:34.891Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-05-15T07:18:34.891896712Z }
file = sled-agent/src/instance.rs:569
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2528 | 2024-05-15T07:18:34.891Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:591
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
new_instance_state = InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-15T07:18:34.891896712Z }
new_vmm_state = VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-05-15T07:18:34.891896712Z }
propolis_id = b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2529 | 2024-05-15T07:18:34.891Z | INFO | SledAgent (InstanceManager): terminating VMM that has exited
file = sled-agent/src/instance.rs:606
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2530 | 2024-05-15T07:18:34.891Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
2531 | 2024-05-15T07:18:34.892Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:381
|
2532 | 2024-05-15T07:18:34.892Z | INFO | SledAgent (ZoneBundler): creating zone bundle
context = ZoneBundleContext { storage_dirs: ["/pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/debug/bundle/zone", "/pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/debug/bundle/zone"], cause: TerminatedInstance, extra_log_dirs: ["/pool/ext/4e6ef30f-a7d9-4acc-99aa-59ae1084fa14/crypt/debug", "/pool/ext/12d428b9-eedd-40c0-8928-1d92a73f181d/crypt/debug", "/pool/ext/08d4507c-e72e-4afd-9f03-5e28f4ad0909/crypt/debug", "/pool/ext/7ba4af02-bf8f-47c6-af79-d69fdad5d4bf/crypt/debug", "/pool/ext/62a187a2-0706-420e-a881-83ff781ad6c1/crypt/debug", "/pool/ext/5da68ecb-08f6-4895-925f-903c81000441/crypt/debug", "/pool/ext/3b36f3f1-29d0-4d11-8b6b-ae2a89620d43/crypt/debug", "/pool/ext/2780ea55-1799-4c3e-916a-9024b9586eaf/crypt/debug", "/pool/ext/bae019e4-4e95-466e-aa8a-f660660e26c1/crypt/debug"] }
file = sled-agent/src/zone_bundle.rs:437
zone_name = oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
|
2533 | 2024-05-15T07:18:35.531Z | INFO | SledAgent (ZoneBundler): finished zone bundle
file = sled-agent/src/zone_bundle.rs:1230
metadata = ZoneBundleMetadata { id: ZoneBundleId { zone_name: "oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543", bundle_id: 54a3329a-dfe0-4d98-904a-0afc9d2e7667 }, time_created: 2024-05-15T07:18:34.892497534Z, version: 0, cause: TerminatedInstance }
|
2534 | 2024-05-15T07:18:35.531Z | WARN | SledAgent (InstanceManager): Halting and removing zone: oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543
file = sled-agent/src/instance.rs:780
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2535 | 2024-05-15T07:18:37.284Z | INFO | SledAgent (InstanceManager): halt_and_remove_logged: Previous zone state: Running
file = illumos-utils/src/zone.rs:269
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|
2536 | 2024-05-15T07:18:37.312Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-15T07:18:34.891896712Z }, propolis_id: b926d0d2-78b1-4d16-b1ed-d24aaeaf7543, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-15T07:18:37.312163043Z } }
|
2537 | 2024-05-15T07:18:37.388Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:50316
|
2538 | 2024-05-15T07:18:37.390Z | WARN | SledAgent (PortManager): unset_virtual_nic_host unimplmented
file = illumos-utils/src/opte/port_manager.rs:620
|
2539 | 2024-05-15T07:18:37.390Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878
latency_us = 1698
local_addr = [fd00:1122:3344:101::1]:12345
method = DELETE
remote_addr = [fd00:1122:3344:101::a]:50316
req_id = e909a894-9f95-4ab3-8e6b-b5686c5b6344
response_code = 204
uri = /v2p/6a12a6a1-9861-4fe8-bb03-f96ecb313186
|
2540 | 2024-05-15T07:18:37.650Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:491
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-15T07:18:34.891896712Z }, propolis_id: b926d0d2-78b1-4d16-b1ed-d24aaeaf7543, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-15T07:18:37.312163043Z } }
|
2541 | 2024-05-15T07:18:37.654Z | INFO | SledAgent (InstanceManager): State monitoring task complete
file = sled-agent/src/instance.rs:723
instance_id = 74b5d93e-2241-4319-8129-fe67108c8f49
|