| 1 | | [ Oct 31 00:12:23 Enabled. ] |
| 2 | | [ Oct 31 00:12:23 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] |
| 3 | | [ Oct 31 00:12:23 Method "start" exited with status 0. ] |
| 4 | | [ Oct 31 00:12:23 Rereading configuration. ] |
| 5 | | [ Oct 31 00:12:23 No 'refresh' method defined. Treating as :true. ] |
| 6 | | note: configured to log to "/dev/stdout" |
| 7 | 2024-10-31T00:12:23.957Z | INFO | SledAgent: Ensuring contract reaper thread
file = illumos-utils/src/running_zone.rs:113
|
| 8 | 2024-10-31T00:12:23.957Z | INFO | SledAgent (BootstrapAgentStartup): finding links ["net0", "net1"]
file = sled-agent/src/bootstrap/pre_server.rs:255
|
| 9 | 2024-10-31T00:12:24.009Z | INFO | SledAgent (BootstrapAgentStartup): Starting mg-ddm service
addr-objs = [AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]
file = sled-agent/src/bootstrap/pre_server.rs:262
|
| 10 | 2024-10-31T00:12:24.009Z | INFO | SledAgent (BootstrapAgentStartup): Importing mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:42
path = /opt/oxide/mg-ddm/pkg/ddm/manifest.xml
|
| 11 | 2024-10-31T00:12:24.123Z | INFO | SledAgent (BootstrapAgentStartup): Setting mg-ddm interfaces
file = sled-agent/src/bootstrap/maghemite.rs:50
interfaces = ("net0/ll" "net1/ll")
|
| 12 | 2024-10-31T00:12:24.132Z | INFO | SledAgent (BootstrapAgentStartup): Enabling mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:56
|
| 13 | 2024-10-31T00:12:24.140Z | INFO | SledAgent (BootstrapAgentStartup): Ensuring zfs key directory exists
file = sled-agent/src/bootstrap/pre_server.rs:272
path = /var/run/oxide/
|
| 14 | 2024-10-31T00:12:24.742Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }
|
| 15 | 2024-10-31T00:12:24.882Z | INFO | SledAgent: Starting KeyManager
file = sled-agent/src/long_running_tasks.rs:126
|
| 16 | 2024-10-31T00:12:24.882Z | INFO | SledAgent: Starting StorageManager
file = sled-agent/src/long_running_tasks.rs:138
|
| 17 | 2024-10-31T00:12:24.882Z | INFO | SledAgent: Starting StorageMonitor
file = sled-agent/src/long_running_tasks.rs:151
|
| 18 | 2024-10-31T00:12:24.882Z | INFO | SledAgent: Starting HardwareManager
file = sled-agent/src/long_running_tasks.rs:173
nongimlet_observed_disks = []
sled_mode = Scrimlet { asic: SoftNpuZone }
|
| 19 | 2024-10-31T00:12:24.882Z | INFO | SledAgent (HardwareManager): Creating HardwareManager
file = sled-hardware/src/illumos/mod.rs:717
|
| 20 | 2024-10-31T00:12:24.882Z | INFO | SledAgent (DumpSetup-worker): DumpSetup poll loop started.
file = sled-agent/src/dump_setup.rs:520
|
| 21 | 2024-10-31T00:12:24.886Z | INFO | SledAgent (HardwareManager): Generated i86pc baseboard Pc { identifier: "sock", model: "i86pc" }
file = sled-hardware/src/illumos/mod.rs:624
|
| 22 | 2024-10-31T00:12:24.886Z | WARN | SledAgent (HardwareManager): Device is not a Gimlet (i86pc), proceeding with null hardware view
file = sled-hardware/src/illumos/mod.rs:766
|
| 23 | 2024-10-31T00:12:24.886Z | INFO | SledAgent: Starting HardwareMonitor
file = sled-agent/src/long_running_tasks.rs:187
|
| 24 | 2024-10-31T00:12:24.886Z | INFO | SledAgent (HardwareManager): Monitoring for hardware updates
file = sled-hardware/src/illumos/mod.rs:825
|
| 25 | 2024-10-31T00:12:24.886Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/m2_0.vdev
|
| 26 | 2024-10-31T00:12:24.886Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
disks = {}
file = sled-agent/src/hardware_monitor.rs:271
underlay_network_info = None
|
| 27 | 2024-10-31T00:12:24.886Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:451
|
| 28 | 2024-10-31T00:12:24.886Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 29 | 2024-10-31T00:12:24.886Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
path = /scratch/m2_0.vdev
|
| 30 | 2024-10-31T00:12:24.897Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev
file = sled-hardware/src/disk.rs:367
|
| 31 | 2024-10-31T00:12:24.897Z | INFO | SledAgent (StorageResources): Formatting zpool with generated ID
file = sled-hardware/src/disk.rs:378
id = 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool)
|
| 32 | 2024-10-31T00:12:24.994Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:173
zpool = ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal }
|
| 33 | 2024-10-31T00:12:25.401Z | INFO | SledAgent (StorageManager): Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }
file = sled-storage/src/manager.rs:451
|
| 34 | 2024-10-31T00:12:25.401Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/m2_1.vdev
|
| 35 | 2024-10-31T00:12:25.401Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 36 | 2024-10-31T00:12:25.401Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 37 | 2024-10-31T00:12:25.401Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:451
|
| 38 | 2024-10-31T00:12:25.403Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 39 | 2024-10-31T00:12:25.403Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
path = /scratch/m2_1.vdev
|
| 40 | 2024-10-31T00:12:25.410Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev
file = sled-hardware/src/disk.rs:367
|
| 41 | 2024-10-31T00:12:25.410Z | INFO | SledAgent (StorageResources): Formatting zpool with generated ID
file = sled-hardware/src/disk.rs:378
id = 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool)
|
| 42 | 2024-10-31T00:12:25.499Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:173
zpool = ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal }
|
| 43 | 2024-10-31T00:12:25.905Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_0.vdev
|
| 44 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 45 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 46 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 47 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:451
|
| 48 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 49 | 2024-10-31T00:12:25.906Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_1.vdev
|
| 50 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 51 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 52 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 53 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:451
|
| 54 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 55 | 2024-10-31T00:12:25.906Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_2.vdev
|
| 56 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 57 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 58 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 59 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:451
|
| 60 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 61 | 2024-10-31T00:12:25.906Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_3.vdev
|
| 62 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:451
|
| 63 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 64 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 65 | 2024-10-31T00:12:25.906Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_4.vdev
|
| 66 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 67 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 68 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:451
|
| 69 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 70 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 71 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 72 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 73 | 2024-10-31T00:12:25.906Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_5.vdev
|
| 74 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 75 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 76 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 77 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:451
|
| 78 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 79 | 2024-10-31T00:12:25.906Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_6.vdev
|
| 80 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 81 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 82 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 83 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:451
|
| 84 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 85 | 2024-10-31T00:12:25.906Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_7.vdev
|
| 86 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:451
|
| 87 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 88 | 2024-10-31T00:12:25.906Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_8.vdev
|
| 89 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 90 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 91 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:451
|
| 92 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 93 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 94 | 2024-10-31T00:12:25.906Z | INFO | SledAgent: Waiting for boot disk
file = sled-agent/src/long_running_tasks.rs:97
|
| 95 | 2024-10-31T00:12:25.906Z | INFO | SledAgent: Found boot disk DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
file = sled-agent/src/long_running_tasks.rs:99
|
| 96 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 97 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 98 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 99 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 100 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 101 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 102 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 103 | 2024-10-31T00:12:25.906Z | INFO | SledAgent: Starting Bootstore
file = sled-agent/src/long_running_tasks.rs:211
|
| 104 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (bootstore): No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 105 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (bootstore): No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 106 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (bootstore): No ledger found. Loading Fsm::State as Uninitialized
file = bootstore/src/schemes/v0/storage.rs:83
peer_id = pc-sock-i86pc
|
| 107 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (bootstore): No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/cluster/bootstore-network-config.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 108 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (bootstore): No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/cluster/bootstore-network-config.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 109 | 2024-10-31T00:12:25.906Z | INFO | SledAgent (bootstore): No ledger found for network config
file = bootstore/src/schemes/v0/storage.rs:146
peer_id = pc-sock-i86pc
|
| 110 | 2024-10-31T00:12:25.906Z | INFO | SledAgent: Starting Bootstore DDMD poller
file = sled-agent/src/long_running_tasks.rs:216
|
| 111 | 2024-10-31T00:12:25.906Z | INFO | SledAgent: Starting ZoneBundler related tasks
file = sled-agent/src/long_running_tasks.rs:231
|
| 112 | 2024-10-31T00:12:25.917Z | INFO | SledAgent (ServiceManager): Creating ServiceManager
file = sled-agent/src/services.rs:733
|
| 113 | 2024-10-31T00:12:25.917Z | INFO | SledAgent (auto-cleanup-task): top of bundle cleanup loop
file = sled-agent/src/zone_bundle.rs:186
next_cleanup = Instant { tv_sec: 1401, tv_nsec: 854183881 }
time_to_next_cleanup = 599.999946979s
|
| 114 | 2024-10-31T00:12:25.917Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 115 | 2024-10-31T00:12:25.917Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3909
|
| 116 | 2024-10-31T00:12:25.917Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/sled-agent-request.json
file = common/src/ledger.rs:188
|
| 117 | 2024-10-31T00:12:25.917Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/sled-agent-request.json
file = common/src/ledger.rs:188
|
| 118 | 2024-10-31T00:12:25.920Z | INFO | SledAgent (dropshot (BootstrapAgent)): listening
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
|
| 119 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (SledAgentSprocketsServer): Started listening
file = sled-agent/src/bootstrap/sprockets_server.rs:51
local_addr = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 120 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (RSS): Injecting RSS configuration: RackInitializeRequest {
trust_quorum_peers: None,
bootstrap_discovery: OnlyOurs,
ntp_servers: [
"0.pool.ntp.org",
],
dns_servers: [
1.1.1.1,
9.9.9.9,
],
internal_services_ip_pool_ranges: [
V4(
Ipv4Range {
first: 10.151.2.164,
last: 10.151.2.173,
},
),
],
external_dns_ips: [
10.151.2.164,
10.151.2.165,
],
external_dns_zone_name: "oxide.test",
external_certificates: "<redacted>",
recovery_silo: RecoverySiloConfig {
silo_name: Name(
"recovery",
),
user_name: UserId(
"recovery",
),
user_password_hash: NewPasswordHash(
PasswordHashString {
string: "$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY",
encoding: B64,
},
),
},
rack_network_config: RackNetworkConfigV2 {
rack_subnet: Ipv6Net {
addr: fd00:1122:3344:100::,
width: 56,
},
infra_ip_first: 10.151.2.174,
infra_ip_last: 10.151.2.174,
ports: [
PortConfigV2 {
routes: [
RouteConfig {
destination: V4(
Ipv4Net {
addr: 0.0.0.0,
width: 0,
},
),
nexthop: 10.151.2.100,
vlan_id: None,
rib_priority: None,
},
],
addresses: [
UplinkAddressConfig {
address: V4(
Ipv4Net {
addr: 10.151.2.174,
width: 24,
},
),
vlan_id: None,
},
],
switch: Switch0,
port: "qsfp0",
uplink_port_speed: Speed40G,
uplink_port_fec: None,
bgp_peers: [],
autoneg: false,
lldp: None,
},
],
bgp: [],
bfd: [],
},
allowed_source_ips: Any,
}
file = sled-agent/src/rack_setup/service.rs:1074
|
| 121 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = internal-dns/resolver/src/resolver.rs:111
|
| 122 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 123 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (RSS): No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/rss-plan-completed.marker
file = common/src/ledger.rs:188
|
| 124 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (RSS): No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/rss-plan-completed.marker
file = common/src/ledger.rs:188
|
| 125 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (RSS): RSS configuration has not been fully applied yet
file = sled-agent/src/rack_setup/service.rs:1137
|
| 126 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 127 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (RSS): No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/rss-sled-plan.json
file = common/src/ledger.rs:188
|
| 128 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (RSS): No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/rss-sled-plan.json
file = common/src/ledger.rs:188
|
| 129 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (RSS): Creating new allocation plan
file = sled-agent/src/rack_setup/service.rs:1179
|
| 130 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (RSS): Serializing plan
file = sled-agent/src/rack_setup/plan/sled.rs:156
|
| 131 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (RSS): Creating plan for the sled at fdb0:18c0:4d0d:a02a::1
file = sled-agent/src/rack_setup/plan/sled.rs:129
|
| 132 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 133 | 2024-10-31T00:12:25.922Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.rss-sled-plan.json.tmp
file = common/src/ledger.rs:199
|
| 134 | 2024-10-31T00:12:25.923Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.rss-sled-plan.json.tmp
file = common/src/ledger.rs:199
|
| 135 | 2024-10-31T00:12:25.923Z | INFO | SledAgent (RSS): Sled plan written to storage: Plan {
rack_id: 91960f7f-96b0-4f89-964d-edd8b80ff639,
sleds: {
[fdb0:18c0:4d0d:a02a::1]:12346: StartSledAgentRequest {
generation: 0,
schema_version: 1,
body: StartSledAgentRequestBody {
id: f95bcc91-ac8f-446a-8a84-3ca7f7459fd7 (sled),
rack_id: 91960f7f-96b0-4f89-964d-edd8b80ff639,
use_trust_quorum: false,
is_lrtq_learner: false,
subnet: Ipv6Subnet {
net: Ipv6Net {
addr: fd00:1122:3344:101::,
width: 64,
},
},
},
},
},
config: RackInitializeRequest {
trust_quorum_peers: None,
bootstrap_discovery: OnlyOurs,
ntp_servers: [
"0.pool.ntp.org",
],
dns_servers: [
1.1.1.1,
9.9.9.9,
],
internal_services_ip_pool_ranges: [
V4(
Ipv4Range {
first: 10.151.2.164,
last: 10.151.2.173,
},
),
],
external_dns_ips: [
10.151.2.164,
10.151.2.165,
],
external_dns_zone_name: "oxide.test",
external_certificates: "<redacted>",
recovery_silo: RecoverySiloConfig {
silo_name: Name(
"recovery",
),
user_name: UserId(
"recovery",
),
user_password_hash: NewPasswordHash(
PasswordHashString {
string: "$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY",
encoding: B64,
},
),
},
rack_network_config: RackNetworkConfigV2 {
rack_subnet: Ipv6Net {
addr: fd00:1122:3344:100::,
width: 56,
},
infra_ip_first: 10.151.2.174,
infra_ip_last: 10.151.2.174,
ports: [
PortConfigV2 {
routes: [
RouteConfig {
destination: V4(
Ipv4Net {
addr: 0.0.0.0,
width: 0,
},
),
nexthop: 10.151.2.100,
vlan_id: None,
rib_priority: None,
},
],
addresses: [
UplinkAddressConfig {
address: V4(
Ipv4Net {
addr: 10.151.2.174,
width: 24,
},
),
vlan_id: None,
},
],
switch: Switch0,
port: "qsfp0",
uplink_port_speed: Speed40G,
uplink_port_fec: None,
bgp_peers: [],
autoneg: false,
lldp: None,
},
],
bgp: [],
bfd: [],
},
allowed_source_ips: Any,
},
}
file = sled-agent/src/rack_setup/plan/sled.rs:176
|
| 136 | 2024-10-31T00:12:25.923Z | INFO | SledAgent (RSS): Writing Rack Network Configuration to bootstore
file = sled-agent/src/rack_setup/service.rs:1213
|
| 137 | 2024-10-31T00:12:25.923Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 1, current_generation: 0
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
| 138 | 2024-10-31T00:12:25.923Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 139 | 2024-10-31T00:12:25.923Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 140 | 2024-10-31T00:12:25.923Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 1
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-sock-i86pc
|
| 141 | 2024-10-31T00:12:25.923Z | INFO | SledAgent (BootstrapAgentRssHandler): Received initialization request from RSS
file = sled-agent/src/bootstrap/rss_handle.rs:232
request = StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: f95bcc91-ac8f-446a-8a84-3ca7f7459fd7 (sled), rack_id: 91960f7f-96b0-4f89-964d-edd8b80ff639, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 142 | 2024-10-31T00:12:25.924Z | INFO | SledAgent (ServiceManager): Enabling switch zone (new)
file = sled-agent/src/services.rs:4206
|
| 143 | 2024-10-31T00:12:25.924Z | INFO | SledAgent (ServiceManager): Starting switch zone
file = sled-agent/src/services.rs:4588
|
| 144 | 2024-10-31T00:12:25.929Z | INFO | SledAgent (SledAgentSprocketsClient): Loaded keys and certs
BootstrapAgentClient = [fdb0:18c0:4d0d:a02a::1]:12346
file = /home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50
|
| 145 | 2024-10-31T00:12:25.935Z | INFO | SledAgent (SledAgentSprocketsClient): Certificate chain verified successfully
BootstrapAgentClient = [fdb0:18c0:4d0d:a02a::1]:12346
file = /home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307
|
| 146 | 2024-10-31T00:12:25.935Z | INFO | SledAgent (SledAgentSprocketsClient): Signature verified successfully
BootstrapAgentClient = [fdb0:18c0:4d0d:a02a::1]:12346
file = /home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376
|
| 147 | 2024-10-31T00:12:25.939Z | INFO | SledAgent (SledAgentSprocketsServer): Certificate chain verified successfully
file = /home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307
|
| 148 | 2024-10-31T00:12:25.939Z | INFO | SledAgent (SledAgentSprocketsServer): Signature verified successfully
file = /home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376
|
| 149 | 2024-10-31T00:12:25.939Z | INFO | SledAgent (SledAgentSprocketsServer): Accepted connection
file = sled-agent/src/bootstrap/sprockets_server.rs:74
remote_addr = [fdb0:18c0:4d0d:a02a::1]:59491
|
| 150 | 2024-10-31T00:12:25.940Z | INFO | SledAgent (SledAgentMain): Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: f95bcc91-ac8f-446a-8a84-3ca7f7459fd7 (sled), rack_id: 91960f7f-96b0-4f89-964d-edd8b80ff639, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }
file = sled-agent/src/bootstrap/server.rs:362
|
| 151 | 2024-10-31T00:12:25.940Z | INFO | SledAgent (SledAgentMain): KeyManager: using hardcoded secret retriever
file = sled-agent/src/bootstrap/server.rs:377
|
| 152 | 2024-10-31T00:12:25.940Z | INFO | SledAgent: setting up sled agent server
file = sled-agent/src/server.rs:44
|
| 153 | 2024-10-31T00:12:25.940Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = internal-dns/resolver/src/resolver.rs:111
|
| 154 | 2024-10-31T00:12:25.940Z | INFO | SledAgent (StorageManager): Received KeyManagerReady
file = sled-storage/src/manager.rs:451
|
| 155 | 2024-10-31T00:12:25.940Z | INFO | SledAgent (StorageManager): Loading storage config
file = sled-storage/src/manager.rs:644
|
| 156 | 2024-10-31T00:12:25.940Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }
|
| 157 | 2024-10-31T00:12:25.940Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = load_disks_ledger
|
| 158 | 2024-10-31T00:12:25.940Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = load_disks_ledger
|
| 159 | 2024-10-31T00:12:25.940Z | INFO | SledAgent (StorageManager): No ledger of physical disks exists
file = sled-storage/src/manager.rs:633
|
| 160 | 2024-10-31T00:12:25.940Z | INFO | SledAgent (StorageManager): KeyManager ready, but no ledger detected
file = sled-storage/src/manager.rs:661
|
| 161 | 2024-10-31T00:12:25.940Z | INFO | SledAgent (StorageManager): automatically managing disks
file = sled-storage/src/manager.rs:440
|
| 162 | 2024-10-31T00:12:25.940Z | INFO | SledAgent (StorageResources): Synchronizing disk managment
file = sled-storage/src/resources.rs:302
|
| 163 | 2024-10-31T00:12:25.940Z | INFO | SledAgent (StorageManager): Successfully synchronized disks without error
file = sled-storage/src/manager.rs:532
result = DisksManagementResult { status: [] }
|
| 164 | 2024-10-31T00:12:25.957Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 165 | 2024-10-31T00:12:25.979Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:325
|
| 166 | 2024-10-31T00:12:25.990Z | INFO | SledAgent: SledAgent::new(..) starting
file = sled-agent/src/sled_agent.rs:401
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 167 | 2024-10-31T00:12:25.990Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 168 | 2024-10-31T00:12:25.990Z | INFO | SledAgent: Requested swap device of size 64 GiB
file = sled-agent/src/sled_agent.rs:413
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 169 | 2024-10-31T00:12:25.990Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 170 | 2024-10-31T00:12:25.993Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 112.224006ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6a24f8b7-2dde-4cd6-83ac-a58e5cc8e1a0", "content-length": "124", "date": "Thu, 31 Oct 2024 00:12:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6a24f8b7-2dde-4cd6-83ac-a58e5cc8e1a0" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 171 | 2024-10-31T00:12:25.997Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:363
|
| 172 | 2024-10-31T00:12:26.002Z | INFO | SledAgent: attempting to create encrypted zvol: name="oxi_39b08bb4-95dc-4ffe-8006-7954c81e10b8/swap", size_gb=64
file = sled-agent/src/swap_device.rs:153
|
| 173 | 2024-10-31T00:12:26.026Z | INFO | SledAgent: successfully created encrypted zvol: name="oxi_39b08bb4-95dc-4ffe-8006-7954c81e10b8/swap", size_gb=64
file = sled-agent/src/swap_device.rs:249
|
| 174 | 2024-10-31T00:12:26.026Z | INFO | SledAgent: adding swap device: swapname="/dev/zvol/dsk/oxi_39b08bb4-95dc-4ffe-8006-7954c81e10b8/swap"
file = sled-agent/src/swap_device.rs:96
|
| 175 | 2024-10-31T00:12:26.031Z | INFO | SledAgent: Mounting backing filesystems
file = sled-agent/src/sled_agent.rs:428
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 176 | 2024-10-31T00:12:26.031Z | INFO | SledAgent (BackingFs): Processing fmd
file = sled-agent/src/backing_fs.rs:128
|
| 177 | 2024-10-31T00:12:26.031Z | INFO | SledAgent (BackingFs): Ensuring dataset oxi_39b08bb4-95dc-4ffe-8006-7954c81e10b8/backing/fmd
file = sled-agent/src/backing_fs.rs:138
|
| 178 | 2024-10-31T00:12:26.105Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 179 | 2024-10-31T00:12:26.106Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 106.278469ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d0d36a10-acd3-449d-8a7f-f13310b5aa32", "content-length": "124", "date": "Thu, 31 Oct 2024 00:12:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d0d36a10-acd3-449d-8a7f-f13310b5aa32" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 180 | 2024-10-31T00:12:26.116Z | INFO | SledAgent (BackingFs): Stopping service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:173
|
| 181 | 2024-10-31T00:12:26.214Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 182 | 2024-10-31T00:12:26.214Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 173.499055ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "25b37a77-573c-4426-a396-179b14560521", "content-length": "124", "date": "Thu, 31 Oct 2024 00:12:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "25b37a77-573c-4426-a396-179b14560521" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 183 | 2024-10-31T00:12:26.335Z | INFO | SledAgent (BackingFs): Mounting oxi_39b08bb4-95dc-4ffe-8006-7954c81e10b8/backing/fmd on /var/fm/fmd
file = sled-agent/src/backing_fs.rs:181
|
| 184 | 2024-10-31T00:12:26.348Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/rsrc
file = sled-agent/src/backing_fs.rs:189
|
| 185 | 2024-10-31T00:12:26.348Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/ckpt
file = sled-agent/src/backing_fs.rs:189
|
| 186 | 2024-10-31T00:12:26.348Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/xprt
file = sled-agent/src/backing_fs.rs:189
|
| 187 | 2024-10-31T00:12:26.348Z | INFO | SledAgent (BackingFs): Starting service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:195
|
| 188 | 2024-10-31T00:12:26.389Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 189 | 2024-10-31T00:12:26.389Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 232.323812ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b040bb8b-c7b6-412f-b864-91a16532603a", "content-length": "124", "date": "Thu, 31 Oct 2024 00:12:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b040bb8b-c7b6-412f-b864-91a16532603a" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 190 | 2024-10-31T00:12:26.622Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 191 | 2024-10-31T00:12:26.623Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 255.245271ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d2417123-1f7a-487e-ab33-877e7d2db035", "content-length": "124", "date": "Thu, 31 Oct 2024 00:12:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d2417123-1f7a-487e-ab33-877e7d2db035" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 192 | 2024-10-31T00:12:26.880Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 193 | 2024-10-31T00:12:26.881Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 155.981284ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "33867e01-3e22-4c2e-9c3f-d3eb3fecce07", "content-length": "124", "date": "Thu, 31 Oct 2024 00:12:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "33867e01-3e22-4c2e-9c3f-d3eb3fecce07" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 194 | 2024-10-31T00:12:27.038Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 195 | 2024-10-31T00:12:27.075Z | INFO | SledAgent: using '[AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]' as data links for xde driver
file = illumos-utils/src/opte/illumos.rs:93
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 196 | 2024-10-31T00:12:27.081Z | INFO | SledAgent (dropshot): listening
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197
local_addr = [fd00:1122:3344:101::1]:40997
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 197 | 2024-10-31T00:12:27.081Z | INFO | SledAgent (internal-dns-resolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = internal-dns/resolver/src/resolver.rs:111
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 198 | 2024-10-31T00:12:27.081Z | INFO | SledAgent (producer-server): starting oximeter metric producer server
address = [fd00:1122:3344:101::1]:40997
file = oximeter/producer/src/lib.rs:283
interval = 30s
producer_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 199 | 2024-10-31T00:12:27.082Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 152.407492ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:391
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 200 | 2024-10-31T00:12:27.083Z | INFO | SledAgent (VmmReservoirManager): 50% of 102952026112 physical ram = 51476013056 bytes)
file = sled-agent/src/vmm_reservoir.rs:267
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 201 | 2024-10-31T00:12:27.083Z | INFO | SledAgent (VmmReservoirManager): Setting reservoir size to 49090 MiB bytes
file = sled-agent/src/vmm_reservoir.rs:275
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 202 | 2024-10-31T00:12:27.083Z | INFO | SledAgent (ServiceManager): sled agent started
file = sled-agent/src/services.rs:917
underlay_address = fd00:1122:3344:101::1
|
| 203 | 2024-10-31T00:12:27.083Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = internal-dns/resolver/src/resolver.rs:111
|
| 204 | 2024-10-31T00:12:27.237Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 589.538382ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:391
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 205 | 2024-10-31T00:12:27.828Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 1.04664005s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:391
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 206 | 2024-10-31T00:12:28.876Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 2.796523639s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:391
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 207 | 2024-10-31T00:12:29.598Z | INFO | SledAgent (ServiceManager): Setting up dendrite service
file = sled-agent/src/services.rs:2664
|
| 208 | 2024-10-31T00:12:29.598Z | INFO | SledAgent (ServiceManager): Setting up lldpd service
file = sled-agent/src/services.rs:2882
|
| 209 | 2024-10-31T00:12:29.598Z | INFO | SledAgent (ServiceManager): Setting up MGS service
file = sled-agent/src/services.rs:2534
|
| 210 | 2024-10-31T00:12:29.598Z | INFO | SledAgent (ServiceManager): Setting up wicketd service
file = sled-agent/src/services.rs:2581
|
| 211 | 2024-10-31T00:12:29.598Z | INFO | SledAgent (ServiceManager): Setting up mgd service
file = sled-agent/src/services.rs:2963
|
| 212 | 2024-10-31T00:12:29.598Z | INFO | SledAgent (ServiceManager): Setting up mg-ddm service
file = sled-agent/src/services.rs:3005
|
| 213 | 2024-10-31T00:12:29.600Z | INFO | SledAgent (ServiceManager): Setting up tfport service
file = sled-agent/src/services.rs:2807
|
| 214 | 2024-10-31T00:12:29.601Z | INFO | SledAgent (ServiceManager): Setting up Simulated SP service
file = sled-agent/src/services.rs:2575
|
| 215 | 2024-10-31T00:12:29.602Z | INFO | SledAgent (ServiceManager): Profile for oxz_switch:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService0'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='::1'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/mgs">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="address" value='[::1]:12225'/>
<propval type="astring" name="id" value='711444b0-abed-4e26-979c-bbe11b97bb20'/>
<propval type="astring" name="rack_id" value='91960f7f-96b0-4f89-964d-edd8b80ff639'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/wicketd">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="address" value='[::1]:12226'/>
<propval type="astring" name="artifact-address" value='[fdb0:18c0:4d0d:a02a::2]:12227'/>
<propval type="astring" name="baseboard-file" value='/opt/oxide/baseboard.json'/>
<propval type="astring" name="mgs-address" value='[::1]:12225'/>
<propval type="astring" name="nexus-proxy-address" value='[::]:12229'/>
<propval type="astring" name="rack-subnet" value='fd00:1122:3344::'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/switch_zone_setup">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="gz_local_link_addr" value='fe80::8:20ff:fefc:11b4'/>
<property type="astring" name="link_local_links">
<astring_list>
<value_node value="tfportrear0_0" />
<value_node value="oxBootstrap0" />
</astring_list>
</property>
<propval type="astring" name="bootstrap_addr" value='fdb0:18c0:4d0d:a02a::2'/>
<propval type="astring" name="bootstrap_vnic" value='oxBootstrap0'/>
<propval type="astring" name="baseboard_info" value='{"type":"pc","identifier":"sock","model":"i86pc"}'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/dendrite">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="sled_id" value='f95bcc91-ac8f-446a-8a84-3ca7f7459fd7'/>
<propval type="astring" name="rack_id" value='91960f7f-96b0-4f89-964d-edd8b80ff639'/>
<propval type="astring" name="address" value='[::1]:12224'/>
<propval type="astring" name="front_ports" value='1'/>
<propval type="astring" name="rear_ports" value='1'/>
<propval type="astring" name="port_config" value='/opt/oxide/dendrite/misc/softnpu_single_sled_config.toml'/>
<propval type="astring" name="mgmt" value='uds'/>
<propval type="astring" name="uds_path" value='/opt/softnpu/stuff'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/tfport">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="host" value='[::1]'/>
<propval type="astring" name="port" value='12224'/>
<propval type="astring" name="flags" value='--sync-only'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/lldpd">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="board_rev" value='softnpu_front_1_rear_1'/>
<propval type="astring" name="scrimlet_id" value='sock'/>
<propval type="astring" name="scrimlet_model" value='i86pc'/>
<propval type="astring" name="address" value='[::1]:12230'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/pumpkind">
</service>
<service version="1" type="service" name="oxide/mgd">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="sled_uuid" value='f95bcc91-ac8f-446a-8a84-3ca7f7459fd7'/>
<propval type="astring" name="rack_uuid" value='91960f7f-96b0-4f89-964d-edd8b80ff639'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/mg-ddm">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="mode" value='transit'/>
<propval type="astring" name="dendrite" value='true'/>
<propval type="astring" name="sled_uuid" value='f95bcc91-ac8f-446a-8a84-3ca7f7459fd7'/>
<propval type="astring" name="rack_uuid" value='91960f7f-96b0-4f89-964d-edd8b80ff639'/>
<propval type="astring" name="interfaces" value='tfportrear0_0/ll'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/uplink">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 216 | 2024-10-31T00:12:29.602Z | INFO | SledAgent (ServiceManager): Booting oxz_switch zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_switch
|
| 217 | 2024-10-31T00:12:29.874Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 32.407876ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 218 | 2024-10-31T00:12:30.090Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 133.257829ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 219 | 2024-10-31T00:12:30.294Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 222.570588ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 220 | 2024-10-31T00:12:30.689Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 473.258107ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 221 | 2024-10-31T00:12:31.331Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 1.057553924s
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 222 | 2024-10-31T00:12:31.674Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 5.765867686s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:391
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 223 | 2024-10-31T00:12:32.533Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 823.371253ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 224 | 2024-10-31T00:12:33.368Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 1.399699892s
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 225 | 2024-10-31T00:12:34.843Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 637.656992ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 226 | 2024-10-31T00:12:35.794Z | INFO | SledAgent (dropshot (SledAgent)): listening
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197
local_addr = [fd00:1122:3344:101::1]:12345
|
| 227 | 2024-10-31T00:12:35.794Z | INFO | SledAgent (SledAgentMain): Sled Agent loaded; recording configuration
file = sled-agent/src/bootstrap/server.rs:432
|
| 228 | 2024-10-31T00:12:35.794Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 229 | 2024-10-31T00:12:35.794Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:199
|
| 230 | 2024-10-31T00:12:35.794Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:199
|
| 231 | 2024-10-31T00:12:35.794Z | INFO | SledAgent (HardwareMonitor): Sled Agent Started
file = sled-agent/src/hardware_monitor.rs:138
|
| 232 | 2024-10-31T00:12:35.794Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
disks = {}
file = sled-agent/src/hardware_monitor.rs:271
underlay_network_info = Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: "qsfp0", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))
|
| 233 | 2024-10-31T00:12:35.794Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3909
|
| 234 | 2024-10-31T00:12:35.794Z | INFO | SledAgent (SledAgentSprocketsServer): Connection closed
file = sled-agent/src/bootstrap/sprockets_server.rs:81
remote_addr = [fdb0:18c0:4d0d:a02a::1]:59491
|
| 235 | 2024-10-31T00:12:36.010Z | INFO | SledAgent (BootstrapAgentRssHandler): Peer agent initialized
file = sled-agent/src/bootstrap/rss_handle.rs:117
peer = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 236 | 2024-10-31T00:12:36.010Z | INFO | SledAgent (BootstrapAgentRssHandler): Initialized sled agent
file = sled-agent/src/bootstrap/rss_handle.rs:252
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 237 | 2024-10-31T00:12:36.010Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 238 | 2024-10-31T00:12:36.010Z | INFO | SledAgent (RSS): No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/rss-service-plan-v5.json
file = common/src/ledger.rs:188
|
| 239 | 2024-10-31T00:12:36.010Z | INFO | SledAgent (RSS): No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/rss-service-plan-v5.json
file = common/src/ledger.rs:188
|
| 240 | 2024-10-31T00:12:36.010Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 241 | 2024-10-31T00:12:36.010Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 242 | 2024-10-31T00:12:36.010Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 243 | 2024-10-31T00:12:36.011Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 244 | 2024-10-31T00:12:36.016Z | INFO | SledAgent (ServiceManager): Re-enabling running switch zone (new address)
file = sled-agent/src/services.rs:4230
new = [fd00:1122:3344:101::2, ::1]
old = [::1]
|
| 245 | 2024-10-31T00:12:36.016Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists
file = sled-agent/src/services.rs:4246
|
| 246 | 2024-10-31T00:12:36.016Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:534
zone = oxz_switch
|
| 247 | 2024-10-31T00:12:36.248Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:65074
|
| 248 | 2024-10-31T00:12:36.248Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 249 | 2024-10-31T00:12:36.248Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 250 | 2024-10-31T00:12:36.248Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 251 | 2024-10-31T00:12:36.248Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 252 | 2024-10-31T00:12:36.249Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 276
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:65074
req_id = 77c78f72-a23c-4c26-972f-6451972d66ac
response_code = 200
uri = /inventory
|
| 253 | 2024-10-31T00:12:36.278Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:39487
|
| 254 | 2024-10-31T00:12:36.279Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 63
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:39487
req_id = 80c63125-c506-4902-aba7-0f092f46b4de
response_code = 200
uri = /sled-role
|
| 255 | 2024-10-31T00:12:36.280Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 256 | 2024-10-31T00:12:36.280Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.rss-service-plan-v5.json.tmp
file = common/src/ledger.rs:199
|
| 257 | 2024-10-31T00:12:36.280Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.rss-service-plan-v5.json.tmp
file = common/src/ledger.rs:199
|
| 258 | 2024-10-31T00:12:36.281Z | INFO | SledAgent (RSS): Service plan written to storage
file = sled-agent/src/rack_setup/plan/service.rs:886
|
| 259 | 2024-10-31T00:12:36.312Z | INFO | SledAgent (RSS): attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, id: 6617d11f-d09d-4ac1-adc2-df58ad02c088, pool_id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, id: 45278ea7-29d3-4d40-9c55-4aa8089d331e, pool_id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, id: 4b77586d-64ae-48a3-96bf-a7ac0c68508e, pool_id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, id: b8667009-7db6-4f53-9319-06981954ebcb, pool_id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, id: f39e3216-44dc-4792-9bc3-55d1e76cb852, pool_id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, id: fac52a67-aa6f-41bb-bfc5-e839c710a2fb, pool_id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, id: 69c3e515-8df9-4d41-8e83-a7a2ff08777a, pool_id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, id: 22e3722e-0394-4395-a899-668661ff1aa4, pool_id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, id: 75ec6d24-5102-4d8c-b644-3d2f3b9572bb, pool_id: b43e5889-7a07-42ec-9599-254e55788daa (zpool) }] }
file = sled-agent/src/rack_setup/service.rs:376
sled_address = [fd00:1122:3344:101::1]:12345
|
| 260 | 2024-10-31T00:12:36.312Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:42737
|
| 261 | 2024-10-31T00:12:36.312Z | INFO | SledAgent: physical disks ensure
file = sled-agent/src/sled_agent.rs:874
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 262 | 2024-10-31T00:12:36.313Z | INFO | SledAgent (StorageManager): Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, id: 6617d11f-d09d-4ac1-adc2-df58ad02c088, pool_id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, id: 45278ea7-29d3-4d40-9c55-4aa8089d331e, pool_id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, id: 4b77586d-64ae-48a3-96bf-a7ac0c68508e, pool_id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, id: b8667009-7db6-4f53-9319-06981954ebcb, pool_id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, id: f39e3216-44dc-4792-9bc3-55d1e76cb852, pool_id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, id: fac52a67-aa6f-41bb-bfc5-e839c710a2fb, pool_id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, id: 69c3e515-8df9-4d41-8e83-a7a2ff08777a, pool_id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, id: 22e3722e-0394-4395-a899-668661ff1aa4, pool_id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, id: 75ec6d24-5102-4d8c-b644-3d2f3b9572bb, pool_id: b43e5889-7a07-42ec-9599-254e55788daa (zpool) }] }, tx: ... }
file = sled-storage/src/manager.rs:451
|
| 263 | 2024-10-31T00:12:36.313Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = omicron_physical_disks_ensure
|
| 264 | 2024-10-31T00:12:36.313Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = omicron_physical_disks_ensure
|
| 265 | 2024-10-31T00:12:36.313Z | INFO | SledAgent (StorageManager): No previously-stored 'requested disks', creating new ledger
file = sled-storage/src/manager.rs:866
request = omicron_physical_disks_ensure
|
| 266 | 2024-10-31T00:12:36.313Z | INFO | SledAgent (StorageResources): Synchronizing disk managment
file = sled-storage/src/resources.rs:302
|
| 267 | 2024-10-31T00:12:36.313Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:342
|
| 268 | 2024-10-31T00:12:36.313Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 269 | 2024-10-31T00:12:36.313Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
path = /scratch/u2_0.vdev
|
| 270 | 2024-10-31T00:12:36.468Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev
file = sled-hardware/src/disk.rs:367
|
| 271 | 2024-10-31T00:12:36.468Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool)
|
| 272 | 2024-10-31T00:12:36.941Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f95bcc91-ac8f-446a-8a84-3ca7f7459fd7)
file = sled-agent/src/probe_manager.rs:231
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 273 | 2024-10-31T00:12:37.159Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 8943919027585428539
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 274 | 2024-10-31T00:12:37.159Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 7027826344887932491
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 275 | 2024-10-31T00:12:37.374Z | INFO | SledAgent (VmmReservoirManager): Finished setting reservoir size to 49090 MiB bytes
file = sled-agent/src/vmm_reservoir.rs:279
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 276 | 2024-10-31T00:12:37.405Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:173
zpool = ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }
|
| 277 | 2024-10-31T00:12:37.413Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:217
|
| 278 | 2024-10-31T00:12:37.413Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:219
|
| 279 | 2024-10-31T00:12:37.413Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:223
|
| 280 | 2024-10-31T00:12:37.413Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:225
|
| 281 | 2024-10-31T00:12:37.413Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 282 | 2024-10-31T00:12:37.413Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 283 | 2024-10-31T00:12:37.413Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:237
|
| 284 | 2024-10-31T00:12:37.443Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 8.09075212s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:391
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 285 | 2024-10-31T00:12:37.448Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 286 | 2024-10-31T00:12:37.457Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_1d6671a7-5622-4671-b7bf-0a757330cf6b/crypt/zone
file = sled-storage/src/dataset.rs:280
|
| 287 | 2024-10-31T00:12:37.631Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_1d6671a7-5622-4671-b7bf-0a757330cf6b
file = sled-storage/src/dataset.rs:366
|
| 288 | 2024-10-31T00:12:37.639Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:440
|
| 289 | 2024-10-31T00:12:37.639Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:357
|
| 290 | 2024-10-31T00:12:37.639Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:342
|
| 291 | 2024-10-31T00:12:37.639Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 292 | 2024-10-31T00:12:37.639Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
path = /scratch/u2_1.vdev
|
| 293 | 2024-10-31T00:12:37.650Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev
file = sled-hardware/src/disk.rs:367
|
| 294 | 2024-10-31T00:12:37.650Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool)
|
| 295 | 2024-10-31T00:12:37.739Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:173
zpool = ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }
|
| 296 | 2024-10-31T00:12:37.746Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:217
|
| 297 | 2024-10-31T00:12:37.746Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:219
|
| 298 | 2024-10-31T00:12:37.746Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:223
|
| 299 | 2024-10-31T00:12:37.746Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:225
|
| 300 | 2024-10-31T00:12:37.746Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 301 | 2024-10-31T00:12:37.746Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 302 | 2024-10-31T00:12:37.746Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:237
|
| 303 | 2024-10-31T00:12:37.780Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 304 | 2024-10-31T00:12:37.787Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_7dd5e852-297b-4341-9eef-72d5112891d5/crypt/zone
file = sled-storage/src/dataset.rs:280
|
| 305 | 2024-10-31T00:12:37.944Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f95bcc91-ac8f-446a-8a84-3ca7f7459fd7)
file = sled-agent/src/probe_manager.rs:231
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 306 | 2024-10-31T00:12:37.958Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_7dd5e852-297b-4341-9eef-72d5112891d5
file = sled-storage/src/dataset.rs:366
|
| 307 | 2024-10-31T00:12:37.966Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:440
|
| 308 | 2024-10-31T00:12:37.966Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:357
|
| 309 | 2024-10-31T00:12:37.966Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:342
|
| 310 | 2024-10-31T00:12:37.966Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 311 | 2024-10-31T00:12:37.966Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
path = /scratch/u2_2.vdev
|
| 312 | 2024-10-31T00:12:37.976Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev
file = sled-hardware/src/disk.rs:367
|
| 313 | 2024-10-31T00:12:37.976Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool)
|
| 314 | 2024-10-31T00:12:38.075Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/dataset.rs:173
zpool = ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }
|
| 315 | 2024-10-31T00:12:38.083Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/dataset.rs:217
|
| 316 | 2024-10-31T00:12:38.083Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:219
|
| 317 | 2024-10-31T00:12:38.083Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:223
|
| 318 | 2024-10-31T00:12:38.083Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:225
|
| 319 | 2024-10-31T00:12:38.083Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 320 | 2024-10-31T00:12:38.083Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 321 | 2024-10-31T00:12:38.083Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:237
|
| 322 | 2024-10-31T00:12:38.117Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 323 | 2024-10-31T00:12:38.125Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_efb62c8a-6541-476c-8447-df5266c7fcb3/crypt/zone
file = sled-storage/src/dataset.rs:280
|
| 324 | 2024-10-31T00:12:38.298Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_efb62c8a-6541-476c-8447-df5266c7fcb3
file = sled-storage/src/dataset.rs:366
|
| 325 | 2024-10-31T00:12:38.306Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:440
|
| 326 | 2024-10-31T00:12:38.306Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:357
|
| 327 | 2024-10-31T00:12:38.306Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:342
|
| 328 | 2024-10-31T00:12:38.306Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 329 | 2024-10-31T00:12:38.306Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
path = /scratch/u2_3.vdev
|
| 330 | 2024-10-31T00:12:38.315Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev
file = sled-hardware/src/disk.rs:367
|
| 331 | 2024-10-31T00:12:38.315Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool)
|
| 332 | 2024-10-31T00:12:38.409Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/dataset.rs:173
zpool = ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }
|
| 333 | 2024-10-31T00:12:38.417Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/dataset.rs:217
|
| 334 | 2024-10-31T00:12:38.417Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:219
|
| 335 | 2024-10-31T00:12:38.417Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:223
|
| 336 | 2024-10-31T00:12:38.417Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:225
|
| 337 | 2024-10-31T00:12:38.417Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 338 | 2024-10-31T00:12:38.418Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 339 | 2024-10-31T00:12:38.418Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:237
|
| 340 | 2024-10-31T00:12:38.453Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 341 | 2024-10-31T00:12:38.460Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_ca55523e-99d5-46dd-9043-66e4f721f4aa/crypt/zone
file = sled-storage/src/dataset.rs:280
|
| 342 | 2024-10-31T00:12:38.627Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_ca55523e-99d5-46dd-9043-66e4f721f4aa
file = sled-storage/src/dataset.rs:366
|
| 343 | 2024-10-31T00:12:38.635Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:440
|
| 344 | 2024-10-31T00:12:38.635Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:357
|
| 345 | 2024-10-31T00:12:38.635Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:342
|
| 346 | 2024-10-31T00:12:38.635Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 347 | 2024-10-31T00:12:38.635Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
path = /scratch/u2_4.vdev
|
| 348 | 2024-10-31T00:12:38.644Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev
file = sled-hardware/src/disk.rs:367
|
| 349 | 2024-10-31T00:12:38.644Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool)
|
| 350 | 2024-10-31T00:12:38.738Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/dataset.rs:173
zpool = ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }
|
| 351 | 2024-10-31T00:12:38.746Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/dataset.rs:217
|
| 352 | 2024-10-31T00:12:38.746Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:219
|
| 353 | 2024-10-31T00:12:38.746Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:223
|
| 354 | 2024-10-31T00:12:38.746Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:225
|
| 355 | 2024-10-31T00:12:38.746Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 356 | 2024-10-31T00:12:38.746Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 357 | 2024-10-31T00:12:38.746Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:237
|
| 358 | 2024-10-31T00:12:38.779Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 359 | 2024-10-31T00:12:38.787Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_67e5554d-50d7-4b6f-b9c3-723d8231ab86/crypt/zone
file = sled-storage/src/dataset.rs:280
|
| 360 | 2024-10-31T00:12:38.946Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f95bcc91-ac8f-446a-8a84-3ca7f7459fd7)
file = sled-agent/src/probe_manager.rs:231
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 361 | 2024-10-31T00:12:38.958Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_67e5554d-50d7-4b6f-b9c3-723d8231ab86
file = sled-storage/src/dataset.rs:366
|
| 362 | 2024-10-31T00:12:38.966Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:440
|
| 363 | 2024-10-31T00:12:38.966Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:357
|
| 364 | 2024-10-31T00:12:38.966Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:342
|
| 365 | 2024-10-31T00:12:38.966Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 366 | 2024-10-31T00:12:38.966Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
path = /scratch/u2_5.vdev
|
| 367 | 2024-10-31T00:12:38.976Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev
file = sled-hardware/src/disk.rs:367
|
| 368 | 2024-10-31T00:12:38.976Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool)
|
| 369 | 2024-10-31T00:12:39.069Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/dataset.rs:173
zpool = ZpoolName { id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool), kind: External }
|
| 370 | 2024-10-31T00:12:39.077Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/dataset.rs:217
|
| 371 | 2024-10-31T00:12:39.077Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:219
|
| 372 | 2024-10-31T00:12:39.077Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:223
|
| 373 | 2024-10-31T00:12:39.077Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:225
|
| 374 | 2024-10-31T00:12:39.077Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 375 | 2024-10-31T00:12:39.077Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 376 | 2024-10-31T00:12:39.077Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:237
|
| 377 | 2024-10-31T00:12:39.111Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 378 | 2024-10-31T00:12:39.119Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354/crypt/zone
file = sled-storage/src/dataset.rs:280
|
| 379 | 2024-10-31T00:12:39.293Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354
file = sled-storage/src/dataset.rs:366
|
| 380 | 2024-10-31T00:12:39.301Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:440
|
| 381 | 2024-10-31T00:12:39.301Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:357
|
| 382 | 2024-10-31T00:12:39.301Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:342
|
| 383 | 2024-10-31T00:12:39.301Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 384 | 2024-10-31T00:12:39.301Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
path = /scratch/u2_6.vdev
|
| 385 | 2024-10-31T00:12:39.310Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev
file = sled-hardware/src/disk.rs:367
|
| 386 | 2024-10-31T00:12:39.310Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool)
|
| 387 | 2024-10-31T00:12:39.409Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/dataset.rs:173
zpool = ZpoolName { id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool), kind: External }
|
| 388 | 2024-10-31T00:12:39.417Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/dataset.rs:217
|
| 389 | 2024-10-31T00:12:39.417Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:219
|
| 390 | 2024-10-31T00:12:39.417Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:223
|
| 391 | 2024-10-31T00:12:39.417Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:225
|
| 392 | 2024-10-31T00:12:39.417Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 393 | 2024-10-31T00:12:39.417Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 394 | 2024-10-31T00:12:39.417Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:237
|
| 395 | 2024-10-31T00:12:39.451Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 396 | 2024-10-31T00:12:39.459Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_a2173f6b-5437-4ef6-a016-ab4b5f89f5d8/crypt/zone
file = sled-storage/src/dataset.rs:280
|
| 397 | 2024-10-31T00:12:39.624Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_a2173f6b-5437-4ef6-a016-ab4b5f89f5d8
file = sled-storage/src/dataset.rs:366
|
| 398 | 2024-10-31T00:12:39.631Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:440
|
| 399 | 2024-10-31T00:12:39.631Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:357
|
| 400 | 2024-10-31T00:12:39.631Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:342
|
| 401 | 2024-10-31T00:12:39.631Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 402 | 2024-10-31T00:12:39.631Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
path = /scratch/u2_7.vdev
|
| 403 | 2024-10-31T00:12:39.640Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev
file = sled-hardware/src/disk.rs:367
|
| 404 | 2024-10-31T00:12:39.640Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool)
|
| 405 | 2024-10-31T00:12:39.736Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/dataset.rs:173
zpool = ZpoolName { id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool), kind: External }
|
| 406 | 2024-10-31T00:12:39.743Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/dataset.rs:217
|
| 407 | 2024-10-31T00:12:39.743Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:219
|
| 408 | 2024-10-31T00:12:39.743Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:223
|
| 409 | 2024-10-31T00:12:39.743Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:225
|
| 410 | 2024-10-31T00:12:39.743Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 411 | 2024-10-31T00:12:39.743Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 412 | 2024-10-31T00:12:39.743Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:237
|
| 413 | 2024-10-31T00:12:39.778Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 414 | 2024-10-31T00:12:39.786Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_a536fa1e-3243-42d2-9446-aa31a45af0ee/crypt/zone
file = sled-storage/src/dataset.rs:280
|
| 415 | 2024-10-31T00:12:39.948Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f95bcc91-ac8f-446a-8a84-3ca7f7459fd7)
file = sled-agent/src/probe_manager.rs:231
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 416 | 2024-10-31T00:12:39.967Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_a536fa1e-3243-42d2-9446-aa31a45af0ee
file = sled-storage/src/dataset.rs:366
|
| 417 | 2024-10-31T00:12:39.975Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:440
|
| 418 | 2024-10-31T00:12:39.975Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:357
|
| 419 | 2024-10-31T00:12:39.975Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:342
|
| 420 | 2024-10-31T00:12:39.975Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 421 | 2024-10-31T00:12:39.975Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
path = /scratch/u2_8.vdev
|
| 422 | 2024-10-31T00:12:39.985Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev
file = sled-hardware/src/disk.rs:367
|
| 423 | 2024-10-31T00:12:39.985Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = b43e5889-7a07-42ec-9599-254e55788daa (zpool)
|
| 424 | 2024-10-31T00:12:40.086Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/dataset.rs:173
zpool = ZpoolName { id: b43e5889-7a07-42ec-9599-254e55788daa (zpool), kind: External }
|
| 425 | 2024-10-31T00:12:40.094Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/dataset.rs:217
|
| 426 | 2024-10-31T00:12:40.094Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:219
|
| 427 | 2024-10-31T00:12:40.094Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:223
|
| 428 | 2024-10-31T00:12:40.094Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:225
|
| 429 | 2024-10-31T00:12:40.094Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 430 | 2024-10-31T00:12:40.094Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 431 | 2024-10-31T00:12:40.094Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:237
|
| 432 | 2024-10-31T00:12:40.129Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 433 | 2024-10-31T00:12:40.137Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_b43e5889-7a07-42ec-9599-254e55788daa/crypt/zone
file = sled-storage/src/dataset.rs:280
|
| 434 | 2024-10-31T00:12:40.315Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_b43e5889-7a07-42ec-9599-254e55788daa
file = sled-storage/src/dataset.rs:366
|
| 435 | 2024-10-31T00:12:40.324Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:440
|
| 436 | 2024-10-31T00:12:40.324Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:357
|
| 437 | 2024-10-31T00:12:40.324Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.omicron-physical-disks.json.tmp
file = common/src/ledger.rs:199
request = omicron_physical_disks_ensure
|
| 438 | 2024-10-31T00:12:40.324Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: b43e5889-7a07-42ec-9599-254e55788daa (zpool), kind: External } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 439 | 2024-10-31T00:12:40.324Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 440 | 2024-10-31T00:12:40.324Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 441 | 2024-10-31T00:12:40.324Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.omicron-physical-disks.json.tmp
file = common/src/ledger.rs:199
request = omicron_physical_disks_ensure
|
| 442 | 2024-10-31T00:12:40.324Z | INFO | SledAgent: physical disks ensure: Updated storage
file = sled-agent/src/sled_agent.rs:878
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 443 | 2024-10-31T00:12:40.324Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 444 | 2024-10-31T00:12:40.324Z | INFO | SledAgent: physical disks ensure: Propagating new generation of disks
file = sled-agent/src/sled_agent.rs:904
generation = Generation(1)
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 445 | 2024-10-31T00:12:40.324Z | INFO | SledAgent: physical disks ensure: Updated storage monitor
file = sled-agent/src/sled_agent.rs:909
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 446 | 2024-10-31T00:12:40.324Z | INFO | SledAgent: physical disks ensure: Updated zone bundler
file = sled-agent/src/sled_agent.rs:914
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 447 | 2024-10-31T00:12:40.324Z | INFO | SledAgent (ProbeManager): use_only_these_disks: Processing new request
file = sled-agent/src/probe_manager.rs:122
gen = Generation(1)
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 448 | 2024-10-31T00:12:40.324Z | INFO | SledAgent: physical disks ensure: Updated probes
file = sled-agent/src/sled_agent.rs:920
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 449 | 2024-10-31T00:12:40.324Z | INFO | SledAgent (InstanceManager): use_only_these_disks: Processing new request
file = sled-agent/src/instance_manager.rs:799
gen = Generation(1)
|
| 450 | 2024-10-31T00:12:40.324Z | INFO | SledAgent: physical disks ensure: Updated instances
file = sled-agent/src/sled_agent.rs:925
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 451 | 2024-10-31T00:12:40.324Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 4012134
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:42737
req_id = d4f3cf3d-c781-4758-ac20-448471935974
response_code = 200
uri = /omicron-physical-disks
|
| 452 | 2024-10-31T00:12:40.356Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 453 | 2024-10-31T00:12:40.356Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:55488
|
| 454 | 2024-10-31T00:12:40.356Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 455 | 2024-10-31T00:12:40.356Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 456 | 2024-10-31T00:12:40.407Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 457 | 2024-10-31T00:12:40.407Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 458 | 2024-10-31T00:12:40.457Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 459 | 2024-10-31T00:12:40.457Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 460 | 2024-10-31T00:12:40.506Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 461 | 2024-10-31T00:12:40.506Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 462 | 2024-10-31T00:12:40.506Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 463 | 2024-10-31T00:12:40.506Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 464 | 2024-10-31T00:12:40.550Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 465 | 2024-10-31T00:12:40.550Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 466 | 2024-10-31T00:12:40.550Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 467 | 2024-10-31T00:12:40.601Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf
file = illumos-utils/src/zone.rs:325
|
| 468 | 2024-10-31T00:12:40.605Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17
file = illumos-utils/src/zone.rs:325
|
| 469 | 2024-10-31T00:12:40.608Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac
file = illumos-utils/src/zone.rs:325
|
| 470 | 2024-10-31T00:12:40.621Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf
file = illumos-utils/src/zone.rs:363
|
| 471 | 2024-10-31T00:12:40.625Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17
file = illumos-utils/src/zone.rs:363
|
| 472 | 2024-10-31T00:12:40.629Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac
file = illumos-utils/src/zone.rs:363
|
| 473 | 2024-10-31T00:12:40.950Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f95bcc91-ac8f-446a-8a84-3ca7f7459fd7)
file = sled-agent/src/probe_manager.rs:231
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 474 | 2024-10-31T00:12:42.153Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService2'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:2::2'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:2::1'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value='[fd00:1122:3344:2::1]:5353'/>
<propval type="astring" name="dns_address" value='[fd00:1122:3344:2::1]:53'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 475 | 2024-10-31T00:12:42.154Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }
|
| 476 | 2024-10-31T00:12:42.630Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService1'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:1::2'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:1::1'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value='[fd00:1122:3344:1::1]:5353'/>
<propval type="astring" name="dns_address" value='[fd00:1122:3344:1::1]:53'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 477 | 2024-10-31T00:12:42.631Z | INFO | SledAgent (ServiceManager): Booting oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17
|
| 478 | 2024-10-31T00:12:43.112Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }
|
| 479 | 2024-10-31T00:12:43.112Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService3'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:3::2'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:3::1'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value='[fd00:1122:3344:3::1]:5353'/>
<propval type="astring" name="dns_address" value='[fd00:1122:3344:3::1]:53'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 480 | 2024-10-31T00:12:43.112Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }
|
| 481 | 2024-10-31T00:12:43.112Z | INFO | SledAgent (ServiceManager): Booting oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf
|
| 482 | 2024-10-31T00:12:43.126Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17") failed: Property not found. retry in 41.290383ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17
|
| 483 | 2024-10-31T00:12:43.126Z | INFO | SledAgent (ServiceManager): Booting oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac
|
| 484 | 2024-10-31T00:12:43.177Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17") failed: Property not found. retry in 147.573602ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17
|
| 485 | 2024-10-31T00:12:43.248Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf") failed: Property not found. retry in 44.50627ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf
|
| 486 | 2024-10-31T00:12:43.254Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac") failed: Property not found. retry in 63.641545ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac
|
| 487 | 2024-10-31T00:12:43.300Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf") failed: Property not found. retry in 68.297955ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf
|
| 488 | 2024-10-31T00:12:43.389Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac") failed: Property not found. retry in 103.188109ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac
|
| 489 | 2024-10-31T00:12:43.410Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf") failed: Property not found. retry in 189.776748ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf
|
| 490 | 2024-10-31T00:12:43.425Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17") failed: Property not found. retry in 176.142148ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17
|
| 491 | 2024-10-31T00:12:43.522Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac") failed: Property not found. retry in 209.555142ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac
|
| 492 | 2024-10-31T00:12:43.615Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf") failed: Property not found. retry in 485.965918ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf
|
| 493 | 2024-10-31T00:12:43.639Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17") failed: Property not found. retry in 201.058046ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17
|
| 494 | 2024-10-31T00:12:43.745Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac") failed: Property not found. retry in 228.147035ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac
|
| 495 | 2024-10-31T00:12:43.850Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17") failed: Property not found. retry in 607.010979ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17
|
| 496 | 2024-10-31T00:12:43.982Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac") failed: Property not found. retry in 995.328547ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac
|
| 497 | 2024-10-31T00:12:44.110Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf") failed: Property not found. retry in 969.440851ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf
|
| 498 | 2024-10-31T00:12:44.468Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17") failed: Property not found. retry in 707.053584ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17
|
| 499 | 2024-10-31T00:12:44.988Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac") failed: Property not found. retry in 647.813617ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac
|
| 500 | 2024-10-31T00:12:45.093Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf") failed: Property not found. retry in 798.103859ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf
|
| 501 | 2024-10-31T00:12:45.184Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17") failed: Property not found. retry in 1.272723009s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17
|
| 502 | 2024-10-31T00:12:45.645Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac") failed: Property not found. retry in 1.138714892s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac
|
| 503 | 2024-10-31T00:12:45.797Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 3558162335736431262
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 504 | 2024-10-31T00:12:45.797Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 10693765855761231549
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 505 | 2024-10-31T00:12:45.900Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf") failed: Property not found. retry in 950.814857ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf
|
| 506 | 2024-10-31T00:12:45.907Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists - OK
file = sled-agent/src/services.rs:4254
|
| 507 | 2024-10-31T00:12:45.907Z | INFO | SledAgent (ServiceManager): Ensuring there is a default route
file = sled-agent/src/services.rs:4264
gateway = fd00:1122:3344:101::1
|
| 508 | 2024-10-31T00:12:45.912Z | INFO | SledAgent (ServiceManager): configuring dendrite service
file = sled-agent/src/services.rs:4343
|
| 509 | 2024-10-31T00:12:46.014Z | INFO | SledAgent (ServiceManager): refreshed dendrite service with new configuration
file = sled-agent/src/services.rs:4391
|
| 510 | 2024-10-31T00:12:46.014Z | INFO | SledAgent (ServiceManager): configuring lldp service
file = sled-agent/src/services.rs:4418
|
| 511 | 2024-10-31T00:12:46.053Z | INFO | SledAgent (ServiceManager): refreshed lldpd service with new configuration
file = sled-agent/src/services.rs:4431
|
| 512 | 2024-10-31T00:12:46.053Z | INFO | SledAgent (ServiceManager): configuring MGS service
file = sled-agent/src/services.rs:4295
|
| 513 | 2024-10-31T00:12:46.107Z | INFO | SledAgent (ServiceManager): refreshed MGS service with new configuration
file = sled-agent/src/services.rs:4337
|
| 514 | 2024-10-31T00:12:46.107Z | INFO | SledAgent (ServiceManager): configuring wicketd
file = sled-agent/src/services.rs:4398
rack_subnet = fd00:1122:3344::
|
| 515 | 2024-10-31T00:12:46.127Z | INFO | SledAgent (ServiceManager): refreshed wicketd service with new configuration
file = sled-agent/src/services.rs:4409
|
| 516 | 2024-10-31T00:12:46.127Z | INFO | SledAgent (ServiceManager): configuring mgd service
file = sled-agent/src/services.rs:4451
|
| 517 | 2024-10-31T00:12:46.194Z | INFO | SledAgent (ServiceManager): refreshed mgd service with new configuration
file = sled-agent/src/services.rs:4483
|
| 518 | 2024-10-31T00:12:46.194Z | INFO | SledAgent (ServiceManager): configuring mg-ddm service
file = sled-agent/src/services.rs:4489
|
| 519 | 2024-10-31T00:12:46.282Z | INFO | SledAgent (ServiceManager): refreshed mg-ddm service with new configuration
file = sled-agent/src/services.rs:4530
|
| 520 | 2024-10-31T00:12:46.282Z | INFO | SledAgent (ServiceManager): Determining physical location of our switch zone at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:370
|
| 521 | 2024-10-31T00:12:46.315Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/f95bcc91-ac8f-446a-8a84-3ca7f7459fd7)
file = sled-agent/src/nexus.rs:527
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 522 | 2024-10-31T00:12:46.315Z | WARN | SledAgent (ServiceManager): Failed to get switch ID from MGS (retrying in 31.056067ms)
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)
file = sled-agent/src/bootstrap/early_networking.rs:389
|
| 523 | 2024-10-31T00:12:46.347Z | WARN | SledAgent (ServiceManager): Failed to get switch ID from MGS (retrying in 102.751123ms)
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)
file = sled-agent/src/bootstrap/early_networking.rs:389
|
| 524 | 2024-10-31T00:12:46.451Z | WARN | SledAgent (ServiceManager): Failed to get switch ID from MGS (retrying in 254.901734ms)
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)
file = sled-agent/src/bootstrap/early_networking.rs:389
|
| 525 | 2024-10-31T00:12:46.464Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17") failed: Property not found. retry in 1.015631829s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17
|
| 526 | 2024-10-31T00:12:46.708Z | WARN | SledAgent (ServiceManager): Failed to get switch ID from MGS (retrying in 470.502072ms)
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)
file = sled-agent/src/bootstrap/early_networking.rs:389
|
| 527 | 2024-10-31T00:12:46.793Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac") failed: Property not found. retry in 1.395533638s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac
|
| 528 | 2024-10-31T00:12:46.858Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf") failed: Property not found. retry in 1.493017033s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf
|
| 529 | 2024-10-31T00:12:47.098Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::2]:35442
|
| 530 | 2024-10-31T00:12:47.098Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 76
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::2]:35442
req_id = 35ce6436-817f-4315-a0b0-d105411bc26b
response_code = 200
uri = /sled-identifiers
|
| 531 | 2024-10-31T00:12:47.162Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 8943919027585428539
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 532 | 2024-10-31T00:12:47.163Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 7027826344887932491
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 533 | 2024-10-31T00:12:47.180Z | INFO | SledAgent (ServiceManager): Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:418
|
| 534 | 2024-10-31T00:12:47.211Z | INFO | SledAgent (ServiceManager): Building Port Configuration
file = sled-agent/src/bootstrap/early_networking.rs:686
|
| 535 | 2024-10-31T00:12:47.211Z | INFO | SledAgent (ServiceManager): Checking dendrite uptime
file = sled-agent/src/bootstrap/early_networking.rs:724
|
| 536 | 2024-10-31T00:12:47.212Z | INFO | SledAgent (ServiceManager): Dendrite online
file = sled-agent/src/bootstrap/early_networking.rs:727
uptime = 1
|
| 537 | 2024-10-31T00:12:47.212Z | INFO | SledAgent (ServiceManager): Configuring default uplink on switch
config = PortSettings {\n links: {\n "0": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}
file = sled-agent/src/bootstrap/early_networking.rs:440
|
| 538 | 2024-10-31T00:12:47.249Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 539 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageManager): Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }
file = sled-storage/src/manager.rs:451
|
| 540 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 541 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 542 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 543 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 544 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 545 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 546 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 547 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 548 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 549 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 550 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 551 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 552 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 553 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 554 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 555 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 556 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 557 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 558 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 559 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 560 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 561 | 2024-10-31T00:12:47.293Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 562 | 2024-10-31T00:12:47.490Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17") failed: Property not found. retry in 1.486159689s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17
|
| 563 | 2024-10-31T00:12:48.995Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf
|
| 564 | 2024-10-31T00:12:48.995Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17
|
| 565 | 2024-10-31T00:12:48.995Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac
|
| 566 | 2024-10-31T00:12:48.995Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 567 | 2024-10-31T00:12:48.995Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 568 | 2024-10-31T00:12:48.995Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 8639371
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:55488
req_id = 7958c44e-f55c-4b36-b5aa-4896457589d4
response_code = 204
uri = /omicron-zones
|
| 569 | 2024-10-31T00:12:48.997Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:1::1]:5353
file = sled-agent/src/rack_setup/service.rs:576
|
| 570 | 2024-10-31T00:12:49.046Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:2::1]:5353
file = sled-agent/src/rack_setup/service.rs:576
|
| 571 | 2024-10-31T00:12:49.093Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:3::1]:5353
file = sled-agent/src/rack_setup/service.rs:576
|
| 572 | 2024-10-31T00:12:49.138Z | INFO | SledAgent (RSS): Configured all DNS servers
file = sled-agent/src/rack_setup/service.rs:612
|
| 573 | 2024-10-31T00:12:49.138Z | INFO | SledAgent (RSS): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:264
|
| 574 | 2024-10-31T00:12:49.169Z | INFO | SledAgent (RSS): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:281
|
| 575 | 2024-10-31T00:12:49.170Z | INFO | SledAgent (RSS): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:306
|
| 576 | 2024-10-31T00:12:49.170Z | INFO | SledAgent (RSS): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:198
|
| 577 | 2024-10-31T00:12:49.199Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 578 | 2024-10-31T00:12:49.199Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:54328
|
| 579 | 2024-10-31T00:12:49.199Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 580 | 2024-10-31T00:12:49.199Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 581 | 2024-10-31T00:12:49.219Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 582 | 2024-10-31T00:12:49.219Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 583 | 2024-10-31T00:12:49.238Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 584 | 2024-10-31T00:12:49.239Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 585 | 2024-10-31T00:12:49.255Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 586 | 2024-10-31T00:12:49.255Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 587 | 2024-10-31T00:12:49.255Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 588 | 2024-10-31T00:12:49.256Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 589 | 2024-10-31T00:12:49.266Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:264
|
| 590 | 2024-10-31T00:12:49.298Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:281
|
| 591 | 2024-10-31T00:12:49.299Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:306
|
| 592 | 2024-10-31T00:12:49.299Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:198
|
| 593 | 2024-10-31T00:12:49.329Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:430
port = Port { inner: PortInner(PortData { name: "opte0", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 214, 36]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }
|
| 594 | 2024-10-31T00:12:49.329Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1260
zone_type = boundary_ntp
|
| 595 | 2024-10-31T00:12:49.330Z | INFO | SledAgent (ServiceManager): no nat entry found for: 10.151.2.169
file = clients/dpd-client/src/lib.rs:83
|
| 596 | 2024-10-31T00:12:49.330Z | INFO | SledAgent (ServiceManager): creating nat entry for: 10.151.2.169
file = clients/dpd-client/src/lib.rs:90
|
| 597 | 2024-10-31T00:12:49.330Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: 10.151.2.169
file = clients/dpd-client/src/lib.rs:118
|
| 598 | 2024-10-31T00:12:49.330Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 599 | 2024-10-31T00:12:49.353Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f
file = illumos-utils/src/zone.rs:325
|
| 600 | 2024-10-31T00:12:49.370Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f
file = illumos-utils/src/zone.rs:363
|
| 601 | 2024-10-31T00:12:49.993Z | INFO | SledAgent (ServiceManager): Profile for oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService4'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::1b'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/chrony-setup">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="allow" value='fd00:1122:3344:100::/56'/>
<propval type="boolean" name="boundary" value='true'/>
<propval type="astring" name="boundary_pool" value='boundary-ntp.control-plane.oxide.internal'/>
<propval type="astring" name="server" value='0.pool.ntp.org'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="1.1.1.1" />
<value_node value="9.9.9.9" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/ntp">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value='opte0'/>
<propval type="astring" name="gateway" value='172.30.3.1'/>
<propval type="astring" name="ip" value='172.30.3.5'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 602 | 2024-10-31T00:12:49.993Z | INFO | SledAgent (ServiceManager): Booting oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f
|
| 603 | 2024-10-31T00:12:50.125Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f") failed: Property not found. retry in 33.821416ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f
|
| 604 | 2024-10-31T00:12:50.166Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f") failed: Property not found. retry in 76.566629ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f
|
| 605 | 2024-10-31T00:12:50.261Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f") failed: Property not found. retry in 248.779017ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f
|
| 606 | 2024-10-31T00:12:50.519Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f") failed: Property not found. retry in 217.398235ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f
|
| 607 | 2024-10-31T00:12:50.747Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f") failed: Property not found. retry in 701.683378ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f
|
| 608 | 2024-10-31T00:12:51.459Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f") failed: Property not found. retry in 740.410465ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f
|
| 609 | 2024-10-31T00:12:52.209Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f") failed: Property not found. retry in 520.74791ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f
|
| 610 | 2024-10-31T00:12:52.739Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f") failed: Property not found. retry in 1.302574104s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f
|
| 611 | 2024-10-31T00:12:54.052Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f") failed: Property not found. retry in 1.444331711s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f
|
| 612 | 2024-10-31T00:12:55.516Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f
|
| 613 | 2024-10-31T00:12:55.516Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 614 | 2024-10-31T00:12:55.516Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 615 | 2024-10-31T00:12:55.517Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 6317691
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:54328
req_id = b4129e32-e289-45d3-9cf2-669d26c554fe
response_code = 204
uri = /omicron-zones
|
| 616 | 2024-10-31T00:12:55.519Z | INFO | SledAgent (RSS): Waiting for rack time synchronization
file = sled-agent/src/rack_setup/service.rs:653
|
| 617 | 2024-10-31T00:12:55.548Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:633
|
| 618 | 2024-10-31T00:12:55.549Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:61456
|
| 619 | 2024-10-31T00:12:55.557Z | ERRO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3862
|
| 620 | 2024-10-31T00:12:55.557Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 7889
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:61456
req_id = 4d6b5789-e775-4931-84ce-6e7f079ba61e
response_code = 500
uri = /timesync
|
| 621 | 2024-10-31T00:12:55.560Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:683
|
| 622 | 2024-10-31T00:12:55.683Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:633
|
| 623 | 2024-10-31T00:12:55.683Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:54538
|
| 624 | 2024-10-31T00:12:55.798Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 3558162335736431262
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 625 | 2024-10-31T00:12:55.799Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 10693765855761231549
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 626 | 2024-10-31T00:12:57.164Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 8943919027585428539
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 627 | 2024-10-31T00:12:57.164Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 7027826344887932491
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 628 | 2024-10-31T00:12:58.216Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 11284587146137120395
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 629 | 2024-10-31T00:12:58.376Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 4145109922441483943
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 630 | 2024-10-31T00:12:58.997Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 16779317648949940276
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 631 | 2024-10-31T00:12:59.690Z | ERRO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3862
|
| 632 | 2024-10-31T00:12:59.690Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 4007080
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:54538
req_id = 636cb7b8-0cb5-4210-9ba2-3af92ad5f894
response_code = 500
uri = /timesync
|
| 633 | 2024-10-31T00:12:59.692Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:683
|
| 634 | 2024-10-31T00:12:59.848Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:633
|
| 635 | 2024-10-31T00:12:59.848Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:41549
|
| 636 | 2024-10-31T00:12:59.857Z | ERRO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3862
|
| 637 | 2024-10-31T00:12:59.857Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 9081
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:41549
req_id = e25627c8-061c-4a3d-ade7-961c0f0d20af
response_code = 500
uri = /timesync
|
| 638 | 2024-10-31T00:12:59.859Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:683
|
| 639 | 2024-10-31T00:13:00.083Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:633
|
| 640 | 2024-10-31T00:13:00.083Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:61557
|
| 641 | 2024-10-31T00:13:00.092Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 8581
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:61557
req_id = d6f089ab-ac89-4951-837d-21bcd495e0cf
response_code = 200
uri = /timesync
|
| 642 | 2024-10-31T00:13:00.093Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:661
|
| 643 | 2024-10-31T00:13:00.093Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:683
|
| 644 | 2024-10-31T00:13:00.240Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:633
|
| 645 | 2024-10-31T00:13:00.240Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:47639
|
| 646 | 2024-10-31T00:13:00.249Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 9310
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:47639
req_id = 35e5a94e-ff1f-40fe-ab65-e2473fa33165
response_code = 200
uri = /timesync
|
| 647 | 2024-10-31T00:13:00.251Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:661
|
| 648 | 2024-10-31T00:13:00.251Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:683
|
| 649 | 2024-10-31T00:13:00.501Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:633
|
| 650 | 2024-10-31T00:13:00.501Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:36004
|
| 651 | 2024-10-31T00:13:00.510Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 8092
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:36004
req_id = d21cb93a-beee-4491-8d24-4bc54102ef51
response_code = 200
uri = /timesync
|
| 652 | 2024-10-31T00:13:00.513Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:661
|
| 653 | 2024-10-31T00:13:00.513Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:683
|
| 654 | 2024-10-31T00:13:00.731Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:633
|
| 655 | 2024-10-31T00:13:00.731Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:51383
|
| 656 | 2024-10-31T00:13:00.738Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 7481
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:51383
req_id = a0d626f9-dc93-40f5-9187-0ad1f49e10b4
response_code = 200
uri = /timesync
|
| 657 | 2024-10-31T00:13:00.740Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:661
|
| 658 | 2024-10-31T00:13:00.740Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:683
|
| 659 | 2024-10-31T00:13:00.945Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:633
|
| 660 | 2024-10-31T00:13:00.945Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:33658
|
| 661 | 2024-10-31T00:13:00.953Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 7389
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:33658
req_id = 6dfda824-053f-44c7-9e28-385ad2564c78
response_code = 200
uri = /timesync
|
| 662 | 2024-10-31T00:13:00.956Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:661
|
| 663 | 2024-10-31T00:13:00.956Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:683
|
| 664 | 2024-10-31T00:13:01.198Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:633
|
| 665 | 2024-10-31T00:13:01.198Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:49980
|
| 666 | 2024-10-31T00:13:01.205Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 7576
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:49980
req_id = 95218e3b-f519-48c6-a56b-9d2099c5503c
response_code = 200
uri = /timesync
|
| 667 | 2024-10-31T00:13:01.208Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:661
|
| 668 | 2024-10-31T00:13:01.208Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:683
|
| 669 | 2024-10-31T00:13:01.033Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:633
|
| 670 | 2024-10-31T00:13:01.034Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:65286
|
| 671 | 2024-10-31T00:13:01.054Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 20406
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:65286
req_id = cc880ceb-087a-4616-a140-cb519048f3cc
response_code = 200
uri = /timesync
|
| 672 | 2024-10-31T00:13:01.058Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 759117325, ip_addr: 45.63.54.13, stratum: 3, ref_time: 1730333580.7595258, correction: -0.048131339 }
file = sled-agent/src/rack_setup/service.rs:661
|
| 673 | 2024-10-31T00:13:01.058Z | INFO | SledAgent (RSS): Finished setting up Internal DNS and NTP
file = sled-agent/src/rack_setup/service.rs:1299
|
| 674 | 2024-10-31T00:13:01.088Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 675 | 2024-10-31T00:13:01.088Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:33601
|
| 676 | 2024-10-31T00:13:01.089Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 677 | 2024-10-31T00:13:01.089Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 678 | 2024-10-31T00:13:01.108Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 679 | 2024-10-31T00:13:01.108Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 680 | 2024-10-31T00:13:01.126Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 681 | 2024-10-31T00:13:01.126Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 682 | 2024-10-31T00:13:01.145Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 683 | 2024-10-31T00:13:01.145Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 684 | 2024-10-31T00:13:01.194Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 685 | 2024-10-31T00:13:01.195Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 686 | 2024-10-31T00:13:01.243Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 687 | 2024-10-31T00:13:01.243Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 688 | 2024-10-31T00:13:01.292Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 689 | 2024-10-31T00:13:01.292Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 690 | 2024-10-31T00:13:01.341Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 691 | 2024-10-31T00:13:01.341Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 692 | 2024-10-31T00:13:01.395Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 693 | 2024-10-31T00:13:01.395Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 694 | 2024-10-31T00:13:01.395Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 695 | 2024-10-31T00:13:01.395Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 696 | 2024-10-31T00:13:01.425Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 697 | 2024-10-31T00:13:01.425Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 698 | 2024-10-31T00:13:01.425Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 699 | 2024-10-31T00:13:01.425Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 700 | 2024-10-31T00:13:01.425Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 701 | 2024-10-31T00:13:01.426Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389", TimeNotSynchronized), ("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173", TimeNotSynchronized), ("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb", TimeNotSynchronized), ("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae", TimeNotSynchronized), ("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0", TimeNotSynchronized)]
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 337036
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33601
req_id = 508b234d-9acc-4597-b58e-4ba4530761b8
response_code = 500
uri = /omicron-zones
|
| 702 | 2024-10-31T00:13:01.426Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "508b234d-9acc-4597-b58e-4ba4530761b8", "content-length": "124", "date": "Thu, 31 Oct 2024 00:13:01 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "508b234d-9acc-4597-b58e-4ba4530761b8" }
file = sled-agent/src/rack_setup/service.rs:496
retry_after = 116.963977ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 703 | 2024-10-31T00:13:01.544Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 704 | 2024-10-31T00:13:01.544Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 705 | 2024-10-31T00:13:01.544Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 706 | 2024-10-31T00:13:01.563Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 707 | 2024-10-31T00:13:01.563Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 708 | 2024-10-31T00:13:01.580Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 709 | 2024-10-31T00:13:01.580Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 710 | 2024-10-31T00:13:01.597Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 711 | 2024-10-31T00:13:01.597Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 712 | 2024-10-31T00:13:01.614Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 713 | 2024-10-31T00:13:01.614Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 714 | 2024-10-31T00:13:01.632Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 715 | 2024-10-31T00:13:01.633Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 716 | 2024-10-31T00:13:01.649Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 717 | 2024-10-31T00:13:01.649Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 718 | 2024-10-31T00:13:01.667Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 719 | 2024-10-31T00:13:01.667Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 720 | 2024-10-31T00:13:01.685Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 721 | 2024-10-31T00:13:01.685Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 722 | 2024-10-31T00:13:01.685Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 723 | 2024-10-31T00:13:01.685Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 724 | 2024-10-31T00:13:01.716Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 725 | 2024-10-31T00:13:01.716Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 726 | 2024-10-31T00:13:01.716Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 727 | 2024-10-31T00:13:01.716Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 728 | 2024-10-31T00:13:01.716Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 729 | 2024-10-31T00:13:01.716Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0", TimeNotSynchronized), ("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb", TimeNotSynchronized), ("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173", TimeNotSynchronized), ("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389", TimeNotSynchronized), ("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae", TimeNotSynchronized)]
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 172003
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33601
req_id = a4e9d1b0-8b0b-4393-aa94-dfedef6b180c
response_code = 500
uri = /omicron-zones
|
| 730 | 2024-10-31T00:13:01.716Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a4e9d1b0-8b0b-4393-aa94-dfedef6b180c", "content-length": "124", "date": "Thu, 31 Oct 2024 00:13:01 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a4e9d1b0-8b0b-4393-aa94-dfedef6b180c" }
file = sled-agent/src/rack_setup/service.rs:496
retry_after = 72.888596ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 731 | 2024-10-31T00:13:01.791Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 732 | 2024-10-31T00:13:01.791Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 733 | 2024-10-31T00:13:01.791Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 734 | 2024-10-31T00:13:01.810Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 735 | 2024-10-31T00:13:01.810Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 736 | 2024-10-31T00:13:01.827Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 737 | 2024-10-31T00:13:01.827Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 738 | 2024-10-31T00:13:01.846Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 739 | 2024-10-31T00:13:01.846Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 740 | 2024-10-31T00:13:01.864Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 741 | 2024-10-31T00:13:01.864Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 742 | 2024-10-31T00:13:01.882Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 743 | 2024-10-31T00:13:01.882Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 744 | 2024-10-31T00:13:01.900Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 745 | 2024-10-31T00:13:01.900Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 746 | 2024-10-31T00:13:01.917Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 747 | 2024-10-31T00:13:01.918Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 748 | 2024-10-31T00:13:01.936Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 749 | 2024-10-31T00:13:01.936Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 750 | 2024-10-31T00:13:01.936Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 751 | 2024-10-31T00:13:01.936Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 752 | 2024-10-31T00:13:01.968Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 753 | 2024-10-31T00:13:01.968Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 754 | 2024-10-31T00:13:01.968Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 755 | 2024-10-31T00:13:01.968Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 756 | 2024-10-31T00:13:01.968Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 757 | 2024-10-31T00:13:01.969Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb", TimeNotSynchronized), ("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389", TimeNotSynchronized), ("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0", TimeNotSynchronized), ("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173", TimeNotSynchronized), ("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae", TimeNotSynchronized)]
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 177707
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33601
req_id = 25fe81dd-7b1f-4f9e-96e9-cdee23d3f23c
response_code = 500
uri = /omicron-zones
|
| 758 | 2024-10-31T00:13:01.969Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "25fe81dd-7b1f-4f9e-96e9-cdee23d3f23c", "content-length": "124", "date": "Thu, 31 Oct 2024 00:13:01 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "25fe81dd-7b1f-4f9e-96e9-cdee23d3f23c" }
file = sled-agent/src/rack_setup/service.rs:496
retry_after = 104.87395ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 759 | 2024-10-31T00:13:02.075Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 760 | 2024-10-31T00:13:02.075Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 761 | 2024-10-31T00:13:02.075Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 762 | 2024-10-31T00:13:02.094Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 763 | 2024-10-31T00:13:02.094Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 764 | 2024-10-31T00:13:02.111Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 765 | 2024-10-31T00:13:02.111Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 766 | 2024-10-31T00:13:02.128Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 767 | 2024-10-31T00:13:02.128Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 768 | 2024-10-31T00:13:02.145Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 769 | 2024-10-31T00:13:02.145Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 770 | 2024-10-31T00:13:02.163Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 771 | 2024-10-31T00:13:02.163Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 772 | 2024-10-31T00:13:02.181Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 773 | 2024-10-31T00:13:02.181Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 774 | 2024-10-31T00:13:02.198Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 775 | 2024-10-31T00:13:02.198Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 776 | 2024-10-31T00:13:02.216Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 777 | 2024-10-31T00:13:02.216Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 778 | 2024-10-31T00:13:02.216Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 779 | 2024-10-31T00:13:02.216Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 780 | 2024-10-31T00:13:02.247Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 781 | 2024-10-31T00:13:02.247Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 782 | 2024-10-31T00:13:02.247Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 783 | 2024-10-31T00:13:02.247Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 784 | 2024-10-31T00:13:02.247Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 785 | 2024-10-31T00:13:02.247Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389", TimeNotSynchronized), ("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0", TimeNotSynchronized), ("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb", TimeNotSynchronized), ("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173", TimeNotSynchronized), ("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae", TimeNotSynchronized)]
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 172783
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33601
req_id = 3bc128d0-3ec9-48a1-af94-66370aa1088b
response_code = 500
uri = /omicron-zones
|
| 786 | 2024-10-31T00:13:02.247Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3bc128d0-3ec9-48a1-af94-66370aa1088b", "content-length": "124", "date": "Thu, 31 Oct 2024 00:13:02 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3bc128d0-3ec9-48a1-af94-66370aa1088b" }
file = sled-agent/src/rack_setup/service.rs:496
retry_after = 229.479744ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 787 | 2024-10-31T00:13:02.477Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 788 | 2024-10-31T00:13:02.477Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 789 | 2024-10-31T00:13:02.477Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 790 | 2024-10-31T00:13:02.496Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 791 | 2024-10-31T00:13:02.496Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 792 | 2024-10-31T00:13:02.514Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 793 | 2024-10-31T00:13:02.514Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 794 | 2024-10-31T00:13:02.532Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 795 | 2024-10-31T00:13:02.532Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 796 | 2024-10-31T00:13:02.550Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 797 | 2024-10-31T00:13:02.550Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 798 | 2024-10-31T00:13:02.569Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 799 | 2024-10-31T00:13:02.569Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 800 | 2024-10-31T00:13:02.587Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 801 | 2024-10-31T00:13:02.587Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 802 | 2024-10-31T00:13:02.605Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 803 | 2024-10-31T00:13:02.605Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 804 | 2024-10-31T00:13:02.624Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 805 | 2024-10-31T00:13:02.624Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 806 | 2024-10-31T00:13:02.624Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 807 | 2024-10-31T00:13:02.624Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 808 | 2024-10-31T00:13:02.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 809 | 2024-10-31T00:13:02.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 810 | 2024-10-31T00:13:02.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 811 | 2024-10-31T00:13:02.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 812 | 2024-10-31T00:13:02.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 813 | 2024-10-31T00:13:02.656Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389", TimeNotSynchronized), ("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae", TimeNotSynchronized), ("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0", TimeNotSynchronized), ("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb", TimeNotSynchronized), ("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173", TimeNotSynchronized)]
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 179621
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33601
req_id = 28029f0f-3d82-4d96-8bdf-5020d0a5230d
response_code = 500
uri = /omicron-zones
|
| 814 | 2024-10-31T00:13:02.656Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "28029f0f-3d82-4d96-8bdf-5020d0a5230d", "content-length": "124", "date": "Thu, 31 Oct 2024 00:13:02 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "28029f0f-3d82-4d96-8bdf-5020d0a5230d" }
file = sled-agent/src/rack_setup/service.rs:496
retry_after = 256.978253ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 815 | 2024-10-31T00:13:02.914Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 816 | 2024-10-31T00:13:02.915Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 817 | 2024-10-31T00:13:02.915Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 818 | 2024-10-31T00:13:02.936Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 819 | 2024-10-31T00:13:02.936Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 820 | 2024-10-31T00:13:02.953Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 821 | 2024-10-31T00:13:02.953Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 822 | 2024-10-31T00:13:02.969Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 823 | 2024-10-31T00:13:02.969Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 824 | 2024-10-31T00:13:02.986Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 825 | 2024-10-31T00:13:02.986Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 826 | 2024-10-31T00:13:03.003Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 827 | 2024-10-31T00:13:03.003Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 828 | 2024-10-31T00:13:03.021Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 829 | 2024-10-31T00:13:03.021Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 830 | 2024-10-31T00:13:03.039Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 831 | 2024-10-31T00:13:03.039Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 832 | 2024-10-31T00:13:03.056Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 833 | 2024-10-31T00:13:03.056Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 834 | 2024-10-31T00:13:03.057Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 835 | 2024-10-31T00:13:03.057Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 836 | 2024-10-31T00:13:03.088Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 837 | 2024-10-31T00:13:03.088Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 838 | 2024-10-31T00:13:03.088Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 839 | 2024-10-31T00:13:03.088Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 840 | 2024-10-31T00:13:03.088Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 841 | 2024-10-31T00:13:03.088Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389", TimeNotSynchronized), ("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae", TimeNotSynchronized), ("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0", TimeNotSynchronized), ("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173", TimeNotSynchronized), ("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb", TimeNotSynchronized)]
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 174553
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33601
req_id = 696a19ff-b5b6-4e2f-85a1-db030f514c74
response_code = 500
uri = /omicron-zones
|
| 842 | 2024-10-31T00:13:03.089Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "696a19ff-b5b6-4e2f-85a1-db030f514c74", "content-length": "124", "date": "Thu, 31 Oct 2024 00:13:02 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "696a19ff-b5b6-4e2f-85a1-db030f514c74" }
file = sled-agent/src/rack_setup/service.rs:496
retry_after = 297.349131ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 843 | 2024-10-31T00:13:03.386Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 844 | 2024-10-31T00:13:03.386Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 845 | 2024-10-31T00:13:03.386Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 846 | 2024-10-31T00:13:03.407Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 847 | 2024-10-31T00:13:03.407Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 848 | 2024-10-31T00:13:03.425Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 849 | 2024-10-31T00:13:03.425Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 850 | 2024-10-31T00:13:03.442Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 851 | 2024-10-31T00:13:03.442Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 852 | 2024-10-31T00:13:03.459Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 853 | 2024-10-31T00:13:03.459Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 854 | 2024-10-31T00:13:03.478Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 855 | 2024-10-31T00:13:03.478Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 856 | 2024-10-31T00:13:03.496Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 857 | 2024-10-31T00:13:03.496Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 858 | 2024-10-31T00:13:03.514Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 859 | 2024-10-31T00:13:03.514Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 860 | 2024-10-31T00:13:03.533Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 861 | 2024-10-31T00:13:03.533Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 862 | 2024-10-31T00:13:03.533Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 863 | 2024-10-31T00:13:03.533Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 864 | 2024-10-31T00:13:03.564Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 865 | 2024-10-31T00:13:03.565Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 866 | 2024-10-31T00:13:03.565Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 867 | 2024-10-31T00:13:03.565Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 868 | 2024-10-31T00:13:03.565Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 869 | 2024-10-31T00:13:03.565Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389", TimeNotSynchronized), ("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb", TimeNotSynchronized), ("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae", TimeNotSynchronized), ("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0", TimeNotSynchronized), ("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173", TimeNotSynchronized)]
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 178836
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33601
req_id = ff5e51e8-4363-4ad5-b885-b17557f93512
response_code = 500
uri = /omicron-zones
|
| 870 | 2024-10-31T00:13:03.565Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "ff5e51e8-4363-4ad5-b885-b17557f93512", "content-length": "124", "date": "Thu, 31 Oct 2024 00:13:02 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "ff5e51e8-4363-4ad5-b885-b17557f93512" }
file = sled-agent/src/rack_setup/service.rs:496
retry_after = 309.748907ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 871 | 2024-10-31T00:13:03.874Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 872 | 2024-10-31T00:13:03.875Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 873 | 2024-10-31T00:13:03.875Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 874 | 2024-10-31T00:13:03.894Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 875 | 2024-10-31T00:13:03.894Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 876 | 2024-10-31T00:13:03.912Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 877 | 2024-10-31T00:13:03.912Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 878 | 2024-10-31T00:13:03.930Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 879 | 2024-10-31T00:13:03.930Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 880 | 2024-10-31T00:13:03.948Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 881 | 2024-10-31T00:13:03.949Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 882 | 2024-10-31T00:13:03.967Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 883 | 2024-10-31T00:13:03.967Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 884 | 2024-10-31T00:13:03.984Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 885 | 2024-10-31T00:13:03.984Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 886 | 2024-10-31T00:13:04.002Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 887 | 2024-10-31T00:13:04.002Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 888 | 2024-10-31T00:13:04.020Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 889 | 2024-10-31T00:13:04.020Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 890 | 2024-10-31T00:13:04.020Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 891 | 2024-10-31T00:13:04.020Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 892 | 2024-10-31T00:13:04.052Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 893 | 2024-10-31T00:13:04.052Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 894 | 2024-10-31T00:13:04.052Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 895 | 2024-10-31T00:13:04.052Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 896 | 2024-10-31T00:13:04.052Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 897 | 2024-10-31T00:13:04.052Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae", TimeNotSynchronized), ("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0", TimeNotSynchronized), ("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173", TimeNotSynchronized), ("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389", TimeNotSynchronized), ("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb", TimeNotSynchronized)]
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 177684
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33601
req_id = 549f77f1-ef36-4b5b-a794-57e450029dea
response_code = 500
uri = /omicron-zones
|
| 898 | 2024-10-31T00:13:04.052Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "549f77f1-ef36-4b5b-a794-57e450029dea", "content-length": "124", "date": "Thu, 31 Oct 2024 00:13:03 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "549f77f1-ef36-4b5b-a794-57e450029dea" }
file = sled-agent/src/rack_setup/service.rs:496
retry_after = 218.139011ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 899 | 2024-10-31T00:13:04.271Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 900 | 2024-10-31T00:13:04.271Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 901 | 2024-10-31T00:13:04.271Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 902 | 2024-10-31T00:13:04.292Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 903 | 2024-10-31T00:13:04.292Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 904 | 2024-10-31T00:13:04.309Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 905 | 2024-10-31T00:13:04.309Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 906 | 2024-10-31T00:13:04.326Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 907 | 2024-10-31T00:13:04.326Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 908 | 2024-10-31T00:13:04.344Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 909 | 2024-10-31T00:13:04.344Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 910 | 2024-10-31T00:13:04.364Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 911 | 2024-10-31T00:13:04.364Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 912 | 2024-10-31T00:13:04.383Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 913 | 2024-10-31T00:13:04.383Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 914 | 2024-10-31T00:13:04.401Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 915 | 2024-10-31T00:13:04.401Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 916 | 2024-10-31T00:13:04.419Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 917 | 2024-10-31T00:13:04.419Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 918 | 2024-10-31T00:13:04.419Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 919 | 2024-10-31T00:13:04.419Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 920 | 2024-10-31T00:13:04.450Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 921 | 2024-10-31T00:13:04.450Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 922 | 2024-10-31T00:13:04.450Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 923 | 2024-10-31T00:13:04.450Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 924 | 2024-10-31T00:13:04.450Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 925 | 2024-10-31T00:13:04.451Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae", TimeNotSynchronized), ("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389", TimeNotSynchronized), ("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173", TimeNotSynchronized), ("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb", TimeNotSynchronized), ("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0", TimeNotSynchronized)]
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 180034
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33601
req_id = 8293b467-e7bd-4fa0-bd09-1022022b0138
response_code = 500
uri = /omicron-zones
|
| 926 | 2024-10-31T00:13:04.451Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "8293b467-e7bd-4fa0-bd09-1022022b0138", "content-length": "124", "date": "Thu, 31 Oct 2024 00:13:04 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "8293b467-e7bd-4fa0-bd09-1022022b0138" }
file = sled-agent/src/rack_setup/service.rs:496
retry_after = 396.171428ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 927 | 2024-10-31T00:13:04.847Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 928 | 2024-10-31T00:13:04.847Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 929 | 2024-10-31T00:13:04.847Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 930 | 2024-10-31T00:13:04.867Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 931 | 2024-10-31T00:13:04.867Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 932 | 2024-10-31T00:13:04.885Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 933 | 2024-10-31T00:13:04.885Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 934 | 2024-10-31T00:13:04.903Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 935 | 2024-10-31T00:13:04.903Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 936 | 2024-10-31T00:13:04.920Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 937 | 2024-10-31T00:13:04.920Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 938 | 2024-10-31T00:13:04.938Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 939 | 2024-10-31T00:13:04.938Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 940 | 2024-10-31T00:13:04.956Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 941 | 2024-10-31T00:13:04.956Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 942 | 2024-10-31T00:13:04.974Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 943 | 2024-10-31T00:13:04.974Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 944 | 2024-10-31T00:13:04.993Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 945 | 2024-10-31T00:13:04.993Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 946 | 2024-10-31T00:13:04.993Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 947 | 2024-10-31T00:13:04.993Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 948 | 2024-10-31T00:13:05.012Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 369.866487ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:424
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 949 | 2024-10-31T00:13:05.027Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 950 | 2024-10-31T00:13:05.027Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 951 | 2024-10-31T00:13:05.027Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 952 | 2024-10-31T00:13:05.027Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 953 | 2024-10-31T00:13:05.027Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 954 | 2024-10-31T00:13:05.027Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0", TimeNotSynchronized), ("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389", TimeNotSynchronized), ("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae", TimeNotSynchronized), ("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173", TimeNotSynchronized), ("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb", TimeNotSynchronized)]
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 180397
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33601
req_id = 9b15d538-eb7f-4f77-a1dd-3dfa41494135
response_code = 500
uri = /omicron-zones
|
| 955 | 2024-10-31T00:13:05.027Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "9b15d538-eb7f-4f77-a1dd-3dfa41494135", "content-length": "124", "date": "Thu, 31 Oct 2024 00:13:05 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "9b15d538-eb7f-4f77-a1dd-3dfa41494135" }
file = sled-agent/src/rack_setup/service.rs:496
retry_after = 741.360785ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 956 | 2024-10-31T00:13:05.767Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 957 | 2024-10-31T00:13:05.767Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 958 | 2024-10-31T00:13:05.767Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 959 | 2024-10-31T00:13:05.786Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 960 | 2024-10-31T00:13:05.786Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 961 | 2024-10-31T00:13:05.804Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 962 | 2024-10-31T00:13:05.804Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 963 | 2024-10-31T00:13:05.822Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 964 | 2024-10-31T00:13:05.822Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 965 | 2024-10-31T00:13:05.842Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 966 | 2024-10-31T00:13:05.842Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 967 | 2024-10-31T00:13:05.859Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 968 | 2024-10-31T00:13:05.859Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 969 | 2024-10-31T00:13:05.877Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 970 | 2024-10-31T00:13:05.877Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 971 | 2024-10-31T00:13:05.896Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 972 | 2024-10-31T00:13:05.896Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 973 | 2024-10-31T00:13:05.913Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 974 | 2024-10-31T00:13:05.913Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 975 | 2024-10-31T00:13:05.914Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 976 | 2024-10-31T00:13:05.914Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 977 | 2024-10-31T00:13:05.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 978 | 2024-10-31T00:13:05.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 979 | 2024-10-31T00:13:05.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 980 | 2024-10-31T00:13:05.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 981 | 2024-10-31T00:13:05.945Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 982 | 2024-10-31T00:13:05.945Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173", TimeNotSynchronized), ("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb", TimeNotSynchronized), ("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae", TimeNotSynchronized), ("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0", TimeNotSynchronized), ("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389", TimeNotSynchronized)]
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 178671
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33601
req_id = 82c67722-582a-41a2-a150-baad949c31f0
response_code = 500
uri = /omicron-zones
|
| 983 | 2024-10-31T00:13:05.946Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "82c67722-582a-41a2-a150-baad949c31f0", "content-length": "124", "date": "Thu, 31 Oct 2024 00:13:05 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "82c67722-582a-41a2-a150-baad949c31f0" }
file = sled-agent/src/rack_setup/service.rs:496
retry_after = 590.303585ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 984 | 2024-10-31T00:13:06.535Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 985 | 2024-10-31T00:13:06.536Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 986 | 2024-10-31T00:13:06.536Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 987 | 2024-10-31T00:13:06.555Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 988 | 2024-10-31T00:13:06.555Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 989 | 2024-10-31T00:13:06.572Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 990 | 2024-10-31T00:13:06.572Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 991 | 2024-10-31T00:13:06.592Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 992 | 2024-10-31T00:13:06.592Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 993 | 2024-10-31T00:13:06.610Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 994 | 2024-10-31T00:13:06.610Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 995 | 2024-10-31T00:13:06.629Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 996 | 2024-10-31T00:13:06.629Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 997 | 2024-10-31T00:13:06.647Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 998 | 2024-10-31T00:13:06.647Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 999 | 2024-10-31T00:13:06.665Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1000 | 2024-10-31T00:13:06.665Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 1001 | 2024-10-31T00:13:06.683Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1002 | 2024-10-31T00:13:06.683Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1003 | 2024-10-31T00:13:06.684Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1004 | 2024-10-31T00:13:06.684Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1005 | 2024-10-31T00:13:06.715Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1006 | 2024-10-31T00:13:06.715Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1007 | 2024-10-31T00:13:06.715Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1008 | 2024-10-31T00:13:06.715Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1009 | 2024-10-31T00:13:06.715Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:3277
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1010 | 2024-10-31T00:13:06.715Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae", TimeNotSynchronized), ("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173", TimeNotSynchronized), ("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb", TimeNotSynchronized), ("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389", TimeNotSynchronized), ("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0", TimeNotSynchronized)]
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938
latency_us = 179904
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33601
req_id = 6b0b9796-24d8-4f1c-a09b-6a57844c6740
response_code = 500
uri = /omicron-zones
|
| 1011 | 2024-10-31T00:13:06.715Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6b0b9796-24d8-4f1c-a09b-6a57844c6740", "content-length": "124", "date": "Thu, 31 Oct 2024 00:13:06 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6b0b9796-24d8-4f1c-a09b-6a57844c6740" }
file = sled-agent/src/rack_setup/service.rs:496
retry_after = 658.081037ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1012 | 2024-10-31T00:13:07.373Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1013 | 2024-10-31T00:13:07.373Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1014 | 2024-10-31T00:13:07.373Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1015 | 2024-10-31T00:13:07.391Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1016 | 2024-10-31T00:13:07.391Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1017 | 2024-10-31T00:13:07.408Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1018 | 2024-10-31T00:13:07.408Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1019 | 2024-10-31T00:13:07.426Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1020 | 2024-10-31T00:13:07.426Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 1021 | 2024-10-31T00:13:07.444Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1022 | 2024-10-31T00:13:07.444Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 1023 | 2024-10-31T00:13:07.462Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1024 | 2024-10-31T00:13:07.462Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 1025 | 2024-10-31T00:13:07.479Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1026 | 2024-10-31T00:13:07.479Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 1027 | 2024-10-31T00:13:07.496Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1028 | 2024-10-31T00:13:07.496Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 1029 | 2024-10-31T00:13:07.512Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1030 | 2024-10-31T00:13:07.512Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1031 | 2024-10-31T00:13:07.513Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1032 | 2024-10-31T00:13:07.513Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1033 | 2024-10-31T00:13:07.592Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1034 | 2024-10-31T00:13:07.592Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1035 | 2024-10-31T00:13:07.592Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1036 | 2024-10-31T00:13:07.592Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1037 | 2024-10-31T00:13:07.592Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1038 | 2024-10-31T00:13:07.686Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
file = illumos-utils/src/zone.rs:325
|
| 1039 | 2024-10-31T00:13:07.690Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
file = illumos-utils/src/zone.rs:325
|
| 1040 | 2024-10-31T00:13:07.695Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
file = illumos-utils/src/zone.rs:325
|
| 1041 | 2024-10-31T00:13:07.699Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
file = illumos-utils/src/zone.rs:325
|
| 1042 | 2024-10-31T00:13:07.703Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
file = illumos-utils/src/zone.rs:325
|
| 1043 | 2024-10-31T00:13:07.707Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
file = illumos-utils/src/zone.rs:363
|
| 1044 | 2024-10-31T00:13:07.712Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
file = illumos-utils/src/zone.rs:363
|
| 1045 | 2024-10-31T00:13:07.716Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
file = illumos-utils/src/zone.rs:363
|
| 1046 | 2024-10-31T00:13:07.720Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
file = illumos-utils/src/zone.rs:363
|
| 1047 | 2024-10-31T00:13:07.725Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
file = illumos-utils/src/zone.rs:363
|
| 1048 | 2024-10-31T00:13:10.628Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService7'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::5'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::5]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='3ff969c4-5131-4f29-bd5a-e8433819d173'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::5]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::5]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1049 | 2024-10-31T00:13:10.628Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService9'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::3'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::3]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='f7bfd91e-056e-4254-b3ef-a9b7a528f9cb'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::3]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::3]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1050 | 2024-10-31T00:13:10.628Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1051 | 2024-10-31T00:13:10.634Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1052 | 2024-10-31T00:13:10.667Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService8'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::6'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::6]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='8e581c4b-8e35-4d00-b402-3a7777b44389'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::6]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::6]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1053 | 2024-10-31T00:13:10.668Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1054 | 2024-10-31T00:13:10.679Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService6'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::4'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::4]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='8f2b3ece-56e1-4bdd-9e3d-b87256e617b0'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::4]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::4]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1055 | 2024-10-31T00:13:10.679Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1056 | 2024-10-31T00:13:10.705Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService5'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::7'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::7]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='24b8d53a-5865-43c7-bd1c-6c377ecc3aae'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::7]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::7]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1057 | 2024-10-31T00:13:10.705Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1058 | 2024-10-31T00:13:10.771Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 43.42209ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1059 | 2024-10-31T00:13:10.780Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 34.948242ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1060 | 2024-10-31T00:13:10.804Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 60.127323ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1061 | 2024-10-31T00:13:10.814Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 36.585947ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1062 | 2024-10-31T00:13:10.820Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 74.627288ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1063 | 2024-10-31T00:13:10.827Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 149.316677ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1064 | 2024-10-31T00:13:11.004Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 36.568801ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1065 | 2024-10-31T00:13:11.068Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 143.527706ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1066 | 2024-10-31T00:13:11.137Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 297.236143ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1067 | 2024-10-31T00:13:11.510Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 121.930576ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1068 | 2024-10-31T00:13:11.566Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 101.334099ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1069 | 2024-10-31T00:13:11.590Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 84.597842ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1070 | 2024-10-31T00:13:11.628Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 185.701814ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1071 | 2024-10-31T00:13:11.680Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 234.138582ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1072 | 2024-10-31T00:13:11.715Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 176.349352ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1073 | 2024-10-31T00:13:11.740Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 230.346306ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1074 | 2024-10-31T00:13:11.770Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 223.760477ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1075 | 2024-10-31T00:13:11.866Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 534.77808ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1076 | 2024-10-31T00:13:11.923Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 540.479083ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1077 | 2024-10-31T00:13:11.957Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 509.702471ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1078 | 2024-10-31T00:13:12.019Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 847.920757ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1079 | 2024-10-31T00:13:12.053Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 412.791447ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1080 | 2024-10-31T00:13:12.437Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 937.465388ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1081 | 2024-10-31T00:13:12.565Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 821.170616ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1082 | 2024-10-31T00:13:12.678Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 442.362989ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1083 | 2024-10-31T00:13:12.736Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 692.463594ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1084 | 2024-10-31T00:13:12.957Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 1.471994534s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1085 | 2024-10-31T00:13:13.175Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 1.162927605s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1086 | 2024-10-31T00:13:13.395Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 635.280391ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1087 | 2024-10-31T00:13:13.410Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 719.69628ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1088 | 2024-10-31T00:13:13.436Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 911.952368ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1089 | 2024-10-31T00:13:14.040Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 923.094134ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1090 | 2024-10-31T00:13:14.137Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 828.897591ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1091 | 2024-10-31T00:13:14.344Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 1.022076588s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1092 | 2024-10-31T00:13:14.355Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 523.684335ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1093 | 2024-10-31T00:13:14.434Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 613.587ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1094 | 2024-10-31T00:13:14.887Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 948.000721ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1095 | 2024-10-31T00:13:14.970Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 988.544471ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1096 | 2024-10-31T00:13:14.977Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 713.96363ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1097 | 2024-10-31T00:13:15.056Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 804.816741ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1098 | 2024-10-31T00:13:15.373Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 1.097058601s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1099 | 2024-10-31T00:13:15.529Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:37613
|
| 1100 | 2024-10-31T00:13:15.529Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 83
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:37613
req_id = 82824b63-3dfb-4644-b532-e9e22bfb5f21
response_code = 200
uri = /baseboard
|
| 1101 | 2024-10-31T00:13:15.698Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 1.182442415s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1102 | 2024-10-31T00:13:15.841Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 548.761215ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1103 | 2024-10-31T00:13:15.867Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 717.843358ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1104 | 2024-10-31T00:13:15.965Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 1.057026619s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1105 | 2024-10-31T00:13:16.401Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 596.235733ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1106 | 2024-10-31T00:13:16.480Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 896.671423ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1107 | 2024-10-31T00:13:16.597Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 626.702913ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1108 | 2024-10-31T00:13:16.890Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 626.401326ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1109 | 2024-10-31T00:13:17.008Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 1.492634928s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1110 | 2024-10-31T00:13:17.029Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 647.574208ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1111 | 2024-10-31T00:13:17.231Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 1.480586716s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1112 | 2024-10-31T00:13:17.383Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 533.340458ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1113 | 2024-10-31T00:13:17.524Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 1.15305792s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1114 | 2024-10-31T00:13:17.683Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 1.495352204s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1115 | 2024-10-31T00:13:17.923Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 1.418740025s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1116 | 2024-10-31T00:13:18.507Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 1.068752458s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1117 | 2024-10-31T00:13:18.685Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 1.00018289s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1118 | 2024-10-31T00:13:18.717Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 903.31878ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1119 | 2024-10-31T00:13:19.184Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 921.605487ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1120 | 2024-10-31T00:13:19.348Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 1.05979003s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1121 | 2024-10-31T00:13:19.583Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 626.363016ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1122 | 2024-10-31T00:13:19.627Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 1.31686853s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1123 | 2024-10-31T00:13:19.691Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 1.192556985s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1124 | 2024-10-31T00:13:20.113Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 665.419241ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1125 | 2024-10-31T00:13:20.217Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 780.412975ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1126 | 2024-10-31T00:13:20.377Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 697.00719ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:424
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 1127 | 2024-10-31T00:13:20.415Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 1.070479075s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1128 | 2024-10-31T00:13:20.786Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 532.472239ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1129 | 2024-10-31T00:13:20.890Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 1.207059896s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1130 | 2024-10-31T00:13:20.950Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 1.430733181s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1131 | 2024-10-31T00:13:21.005Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 1.408485635s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1132 | 2024-10-31T00:13:21.328Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 798.697377ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1133 | 2024-10-31T00:13:21.494Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 1.381526725s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1134 | 2024-10-31T00:13:22.106Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 987.874943ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1135 | 2024-10-31T00:13:22.136Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 814.951189ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1136 | 2024-10-31T00:13:22.390Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 831.438053ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1137 | 2024-10-31T00:13:22.423Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 1.303158127s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1138 | 2024-10-31T00:13:22.885Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 742.45015ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1139 | 2024-10-31T00:13:22.961Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 533.302944ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1140 | 2024-10-31T00:13:23.102Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 1.344403668s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1141 | 2024-10-31T00:13:23.230Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 1.239709783s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1142 | 2024-10-31T00:13:23.504Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 1.092834353s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1143 | 2024-10-31T00:13:23.638Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 686.517317ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1144 | 2024-10-31T00:13:23.737Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 1.431567739s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1145 | 2024-10-31T00:13:24.334Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 734.562636ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1146 | 2024-10-31T00:13:24.457Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 1.071300339s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1147 | 2024-10-31T00:13:24.479Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 1.135539718s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1148 | 2024-10-31T00:13:24.606Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 1.337956155s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1149 | 2024-10-31T00:13:25.078Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 881.369237ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1150 | 2024-10-31T00:13:25.178Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 1.401986116s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1151 | 2024-10-31T00:13:25.537Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 554.231768ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1152 | 2024-10-31T00:13:25.624Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 1.239966836s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1153 | 2024-10-31T00:13:25.955Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 659.346355ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1154 | 2024-10-31T00:13:25.967Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 733.100671ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1155 | 2024-10-31T00:13:26.101Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 702.348301ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1156 | 2024-10-31T00:13:26.588Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 1.000240526s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1157 | 2024-10-31T00:13:26.621Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 1.467116707s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1158 | 2024-10-31T00:13:26.709Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 904.13823ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1159 | 2024-10-31T00:13:26.812Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 1.478066418s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1160 | 2024-10-31T00:13:26.872Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 1.42024481s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1161 | 2024-10-31T00:13:27.597Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 827.626919ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1162 | 2024-10-31T00:13:27.621Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 1.341155272s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1163 | 2024-10-31T00:13:28.097Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 1.204592974s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1164 | 2024-10-31T00:13:28.297Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 1.414495839s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1165 | 2024-10-31T00:13:28.304Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 918.253074ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1166 | 2024-10-31T00:13:28.432Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 857.55077ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1167 | 2024-10-31T00:13:28.970Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 1.00362862s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1168 | 2024-10-31T00:13:29.232Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 723.124679ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1169 | 2024-10-31T00:13:29.298Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 738.02967ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1170 | 2024-10-31T00:13:29.311Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0") failed: Property not found. retry in 1.392654421s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1171 | 2024-10-31T00:13:29.721Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389") failed: Property not found. retry in 907.821825ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1172 | 2024-10-31T00:13:29.965Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb") failed: Property not found. retry in 1.471576132s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1173 | 2024-10-31T00:13:29.985Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae") failed: Property not found. retry in 938.713485ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1174 | 2024-10-31T00:13:30.049Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173") failed: Property not found. retry in 1.406370689s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1175 | 2024-10-31T00:13:31.480Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae
|
| 1176 | 2024-10-31T00:13:31.480Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0
|
| 1177 | 2024-10-31T00:13:31.480Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173
|
| 1178 | 2024-10-31T00:13:31.480Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389
|
| 1179 | 2024-10-31T00:13:31.480Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb
|
| 1180 | 2024-10-31T00:13:31.480Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 1181 | 2024-10-31T00:13:31.480Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 1182 | 2024-10-31T00:13:31.480Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 24140658
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33601
req_id = 533a8f26-af11-42d1-870f-cf9b39f9b7bc
response_code = 204
uri = /omicron-zones
|
| 1183 | 2024-10-31T00:13:31.513Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:34741
|
| 1184 | 2024-10-31T00:13:31.542Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::7]:32221
file = sled-agent/src/services.rs:3705
|
| 1185 | 2024-10-31T00:13:47.145Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3723
|
| 1186 | 2024-10-31T00:14:02.219Z | INFO | SledAgent (ServiceManager): Formatting CRDB - Completed
file = sled-agent/src/services.rs:3746
|
| 1187 | 2024-10-31T00:14:02.222Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 30709252
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:34741
req_id = c17b84f1-d712-4b6b-8f55-13cc5431f215
response_code = 204
uri = /cockroachdb
|
| 1188 | 2024-10-31T00:14:02.224Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:55643
|
| 1189 | 2024-10-31T00:14:02.227Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 983.498144ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:424
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 1190 | 2024-10-31T00:14:02.261Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name("ntp-14078d86-af25-4a11-bd50-933b6f13166f"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7 (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea2ccb0-4d8d-450a-b5fa-20f11d99c6b4, kind: Service { id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7 }, name: Name("external-dns-7ed51b56-e87d-4a9a-b9bc-27ece36a86d7"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2beb08a-ae3f-469f-86c4-36645336e7c7 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 486dd504-bd2f-498e-8990-b1d7d833ab10, kind: Service { id: b2beb08a-ae3f-469f-86c4-36645336e7c7 }, name: Name("external-dns-b2beb08a-ae3f-469f-86c4-36645336e7c7"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 100])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1e4bfc38-893c-408c-9d22-948390ba9d86 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1e6d66a8-381f-4d1d-a719-7f1f1a2fab2f, kind: Service { id: 1e4bfc38-893c-408c-9d22-948390ba9d86 }, name: Name("nexus-1e4bfc38-893c-408c-9d22-948390ba9d86"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 239, 133])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2c60990c-614d-44ea-a7bf-2bae04edc1b3 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 397cb5ae-79f3-4840-95d1-a308d95a2fad, kind: Service { id: 2c60990c-614d-44ea-a7bf-2bae04edc1b3 }, name: Name("nexus-2c60990c-614d-44ea-a7bf-2bae04edc1b3"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 222])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1f4e8f94-7bc7-4b29-b92c-ee717847f51c (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: f546a11c-0e59-4819-8b6f-c01e0cfa393a, kind: Service { id: 1f4e8f94-7bc7-4b29-b92c-ee717847f51c }, name: Name("nexus-1f4e8f94-7bc7-4b29-b92c-ee717847f51c"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4ce79729-70af-4ea9-8907-92eccad979f8 (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 12eca9ce-450a-47c6-aa79-35164c1f047f (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8377dba6-b489-43b3-98e1-05bb7df98988 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 065751ee-697c-4a72-8cf7-687dac3ede9c (service), filesystem_pool: Some(ZpoolName { id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 511547b5-244c-4c59-b939-8d95795a2c87 (service), filesystem_pool: Some(ZpoolName { id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a4913a46-2b97-4d0c-965d-3da766243586 (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: e2b91b21-636b-4131-93b7-4d20d375703a (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ee7d195-9a59-4752-ad62-863aa8d03aa4 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32137949-0a38-4d1e-a737-2d07177c3738 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f59615c-c60c-43bf-8c6b-f46c94caeddb (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1e02403f-00b9-457b-a823-4da05e840386 (service), filesystem_pool: Some(ZpoolName { id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool), kind: External } } } }, OmicronZoneConfig { id: d634556d-8226-4df6-8dfc-09c66049e16b (service), filesystem_pool: Some(ZpoolName { id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: c559bdc6-d957-40c2-ad54-ae7947822ea7 (service), filesystem_pool: Some(ZpoolName { id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool), kind: External } } } }, OmicronZoneConfig { id: dde78775-b16f-45ca-91b2-9608c686b32c (service), filesystem_pool: Some(ZpoolName { id: b43e5889-7a07-42ec-9599-254e55788daa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b43e5889-7a07-42ec-9599-254e55788daa (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:459
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1191 | 2024-10-31T00:14:02.261Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:44733
|
| 1192 | 2024-10-31T00:14:02.263Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1193 | 2024-10-31T00:14:02.263Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1194 | 2024-10-31T00:14:02.290Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1195 | 2024-10-31T00:14:02.290Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1196 | 2024-10-31T00:14:02.310Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1197 | 2024-10-31T00:14:02.310Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 1198 | 2024-10-31T00:14:02.331Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1199 | 2024-10-31T00:14:02.331Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 1200 | 2024-10-31T00:14:02.349Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1201 | 2024-10-31T00:14:02.349Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 1202 | 2024-10-31T00:14:02.371Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1203 | 2024-10-31T00:14:02.371Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 1204 | 2024-10-31T00:14:02.392Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1205 | 2024-10-31T00:14:02.392Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 1206 | 2024-10-31T00:14:02.411Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1207 | 2024-10-31T00:14:02.411Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 1208 | 2024-10-31T00:14:02.429Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1209 | 2024-10-31T00:14:02.429Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1210 | 2024-10-31T00:14:02.554Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b2beb08a-ae3f-469f-86c4-36645336e7c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1211 | 2024-10-31T00:14:02.554Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: b2beb08a-ae3f-469f-86c4-36645336e7c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }
|
| 1212 | 2024-10-31T00:14:02.715Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 12eca9ce-450a-47c6-aa79-35164c1f047f, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Clickhouse }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1213 | 2024-10-31T00:14:02.715Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 12eca9ce-450a-47c6-aa79-35164c1f047f, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Clickhouse }, responder: ... }
|
| 1214 | 2024-10-31T00:14:02.825Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a4913a46-2b97-4d0c-965d-3da766243586, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1215 | 2024-10-31T00:14:02.825Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: a4913a46-2b97-4d0c-965d-3da766243586, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1216 | 2024-10-31T00:14:02.965Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: e2b91b21-636b-4131-93b7-4d20d375703a, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1217 | 2024-10-31T00:14:02.965Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: e2b91b21-636b-4131-93b7-4d20d375703a, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1218 | 2024-10-31T00:14:03.105Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ee7d195-9a59-4752-ad62-863aa8d03aa4, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1219 | 2024-10-31T00:14:03.105Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 1ee7d195-9a59-4752-ad62-863aa8d03aa4, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1220 | 2024-10-31T00:14:03.237Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 32137949-0a38-4d1e-a737-2d07177c3738, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1221 | 2024-10-31T00:14:03.237Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 32137949-0a38-4d1e-a737-2d07177c3738, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1222 | 2024-10-31T00:14:03.351Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f59615c-c60c-43bf-8c6b-f46c94caeddb, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1223 | 2024-10-31T00:14:03.351Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 7f59615c-c60c-43bf-8c6b-f46c94caeddb, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1224 | 2024-10-31T00:14:03.465Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e02403f-00b9-457b-a823-4da05e840386, dataset_name: DatasetName { pool_name: ZpoolName { id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1225 | 2024-10-31T00:14:03.465Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: 1e02403f-00b9-457b-a823-4da05e840386, dataset_name: DatasetName { pool_name: ZpoolName { id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1226 | 2024-10-31T00:14:03.590Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: d634556d-8226-4df6-8dfc-09c66049e16b, dataset_name: DatasetName { pool_name: ZpoolName { id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1227 | 2024-10-31T00:14:03.590Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: d634556d-8226-4df6-8dfc-09c66049e16b, dataset_name: DatasetName { pool_name: ZpoolName { id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1228 | 2024-10-31T00:14:03.730Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c559bdc6-d957-40c2-ad54-ae7947822ea7, dataset_name: DatasetName { pool_name: ZpoolName { id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1229 | 2024-10-31T00:14:03.730Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: c559bdc6-d957-40c2-ad54-ae7947822ea7, dataset_name: DatasetName { pool_name: ZpoolName { id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1230 | 2024-10-31T00:14:03.871Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: dde78775-b16f-45ca-91b2-9608c686b32c, dataset_name: DatasetName { pool_name: ZpoolName { id: b43e5889-7a07-42ec-9599-254e55788daa (zpool), kind: External }, kind: Crucible }, responder: ... })
file = sled-storage/src/manager.rs:451
|
| 1231 | 2024-10-31T00:14:03.871Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1059
request = NewFilesystemRequest { dataset_id: dde78775-b16f-45ca-91b2-9608c686b32c, dataset_name: DatasetName { pool_name: ZpoolName { id: b43e5889-7a07-42ec-9599-254e55788daa (zpool), kind: External }, kind: Crucible }, responder: ... }
|
| 1232 | 2024-10-31T00:14:04.012Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1233 | 2024-10-31T00:14:04.013Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1234 | 2024-10-31T00:14:04.013Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1235 | 2024-10-31T00:14:04.014Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1236 | 2024-10-31T00:14:04.135Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:264
|
| 1237 | 2024-10-31T00:14:04.171Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:264
|
| 1238 | 2024-10-31T00:14:04.171Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1239 | 2024-10-31T00:14:04.171Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1240 | 2024-10-31T00:14:04.222Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:264
|
| 1241 | 2024-10-31T00:14:04.222Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1242 | 2024-10-31T00:14:04.222Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1243 | 2024-10-31T00:14:04.222Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1244 | 2024-10-31T00:14:04.222Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1245 | 2024-10-31T00:14:04.222Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1246 | 2024-10-31T00:14:04.388Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:264
|
| 1247 | 2024-10-31T00:14:04.388Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1248 | 2024-10-31T00:14:04.388Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1249 | 2024-10-31T00:14:04.388Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1250 | 2024-10-31T00:14:04.388Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1251 | 2024-10-31T00:14:04.390Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:264
|
| 1252 | 2024-10-31T00:14:04.427Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1253 | 2024-10-31T00:14:04.427Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1254 | 2024-10-31T00:14:04.427Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1255 | 2024-10-31T00:14:04.428Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
file = illumos-utils/src/zone.rs:325
|
| 1256 | 2024-10-31T00:14:04.433Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
file = illumos-utils/src/zone.rs:325
|
| 1257 | 2024-10-31T00:14:04.439Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
file = illumos-utils/src/zone.rs:325
|
| 1258 | 2024-10-31T00:14:04.444Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
file = illumos-utils/src/zone.rs:325
|
| 1259 | 2024-10-31T00:14:04.449Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
file = illumos-utils/src/zone.rs:325
|
| 1260 | 2024-10-31T00:14:04.454Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
file = illumos-utils/src/zone.rs:325
|
| 1261 | 2024-10-31T00:14:04.460Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
file = illumos-utils/src/zone.rs:325
|
| 1262 | 2024-10-31T00:14:04.607Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
file = illumos-utils/src/zone.rs:363
|
| 1263 | 2024-10-31T00:14:04.611Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
file = illumos-utils/src/zone.rs:363
|
| 1264 | 2024-10-31T00:14:04.616Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
file = illumos-utils/src/zone.rs:363
|
| 1265 | 2024-10-31T00:14:04.620Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
file = illumos-utils/src/zone.rs:363
|
| 1266 | 2024-10-31T00:14:04.625Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
file = illumos-utils/src/zone.rs:363
|
| 1267 | 2024-10-31T00:14:04.629Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
file = illumos-utils/src/zone.rs:363
|
| 1268 | 2024-10-31T00:14:04.634Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
file = illumos-utils/src/zone.rs:363
|
| 1269 | 2024-10-31T00:14:04.640Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
file = illumos-utils/src/zone.rs:325
|
| 1270 | 2024-10-31T00:14:04.646Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
file = illumos-utils/src/zone.rs:325
|
| 1271 | 2024-10-31T00:14:04.651Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
file = illumos-utils/src/zone.rs:325
|
| 1272 | 2024-10-31T00:14:04.656Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
file = illumos-utils/src/zone.rs:325
|
| 1273 | 2024-10-31T00:14:04.662Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
file = illumos-utils/src/zone.rs:325
|
| 1274 | 2024-10-31T00:14:04.668Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
file = illumos-utils/src/zone.rs:325
|
| 1275 | 2024-10-31T00:14:04.672Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
file = illumos-utils/src/zone.rs:325
|
| 1276 | 2024-10-31T00:14:04.714Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:281
|
| 1277 | 2024-10-31T00:14:04.714Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
file = illumos-utils/src/zone.rs:363
|
| 1278 | 2024-10-31T00:14:04.720Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
file = illumos-utils/src/zone.rs:363
|
| 1279 | 2024-10-31T00:14:04.724Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
file = illumos-utils/src/zone.rs:363
|
| 1280 | 2024-10-31T00:14:04.728Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
file = illumos-utils/src/zone.rs:363
|
| 1281 | 2024-10-31T00:14:04.733Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
file = illumos-utils/src/zone.rs:363
|
| 1282 | 2024-10-31T00:14:04.738Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
file = illumos-utils/src/zone.rs:363
|
| 1283 | 2024-10-31T00:14:04.743Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
file = illumos-utils/src/zone.rs:363
|
| 1284 | 2024-10-31T00:14:04.786Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:281
|
| 1285 | 2024-10-31T00:14:04.823Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:281
|
| 1286 | 2024-10-31T00:14:04.881Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:281
|
| 1287 | 2024-10-31T00:14:04.938Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:281
|
| 1288 | 2024-10-31T00:14:04.941Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:306
|
| 1289 | 2024-10-31T00:14:04.941Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:198
|
| 1290 | 2024-10-31T00:14:04.992Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:430
port = Port { inner: PortInner(PortData { name: "opte1", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 145, 100]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }
|
| 1291 | 2024-10-31T00:14:04.992Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1260
zone_type = external_dns
|
| 1292 | 2024-10-31T00:14:04.993Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:306
|
| 1293 | 2024-10-31T00:14:04.993Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:198
|
| 1294 | 2024-10-31T00:14:05.048Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:430
port = Port { inner: PortInner(PortData { name: "opte2", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 239, 133]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }
|
| 1295 | 2024-10-31T00:14:05.048Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1260
zone_type = nexus
|
| 1296 | 2024-10-31T00:14:05.050Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:306
|
| 1297 | 2024-10-31T00:14:05.050Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:198
|
| 1298 | 2024-10-31T00:14:05.114Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:430
port = Port { inner: PortInner(PortData { name: "opte3", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 193, 148]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }
|
| 1299 | 2024-10-31T00:14:05.114Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1260
zone_type = nexus
|
| 1300 | 2024-10-31T00:14:05.116Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:306
|
| 1301 | 2024-10-31T00:14:05.116Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:198
|
| 1302 | 2024-10-31T00:14:05.180Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:430
port = Port { inner: PortInner(PortData { name: "opte4", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 244, 222]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }
|
| 1303 | 2024-10-31T00:14:05.180Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1260
zone_type = nexus
|
| 1304 | 2024-10-31T00:14:05.181Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:306
|
| 1305 | 2024-10-31T00:14:05.181Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:198
|
| 1306 | 2024-10-31T00:14:05.235Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:430
port = Port { inner: PortInner(PortData { name: "opte5", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 150, 109]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }
|
| 1307 | 2024-10-31T00:14:05.235Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1260
zone_type = external_dns
|
| 1308 | 2024-10-31T00:14:05.243Z | INFO | SledAgent (ServiceManager): no nat entry found for: 10.151.2.164
file = clients/dpd-client/src/lib.rs:83
|
| 1309 | 2024-10-31T00:14:05.243Z | INFO | SledAgent (ServiceManager): creating nat entry for: 10.151.2.164
file = clients/dpd-client/src/lib.rs:90
|
| 1310 | 2024-10-31T00:14:05.243Z | INFO | SledAgent (ServiceManager): no nat entry found for: 10.151.2.165
file = clients/dpd-client/src/lib.rs:83
|
| 1311 | 2024-10-31T00:14:05.243Z | INFO | SledAgent (ServiceManager): creating nat entry for: 10.151.2.165
file = clients/dpd-client/src/lib.rs:90
|
| 1312 | 2024-10-31T00:14:05.243Z | INFO | SledAgent (ServiceManager): no nat entry found for: 10.151.2.166
file = clients/dpd-client/src/lib.rs:83
|
| 1313 | 2024-10-31T00:14:05.243Z | INFO | SledAgent (ServiceManager): creating nat entry for: 10.151.2.166
file = clients/dpd-client/src/lib.rs:90
|
| 1314 | 2024-10-31T00:14:05.243Z | INFO | SledAgent (ServiceManager): no nat entry found for: 10.151.2.168
file = clients/dpd-client/src/lib.rs:83
|
| 1315 | 2024-10-31T00:14:05.243Z | INFO | SledAgent (ServiceManager): creating nat entry for: 10.151.2.168
file = clients/dpd-client/src/lib.rs:90
|
| 1316 | 2024-10-31T00:14:05.243Z | INFO | SledAgent (ServiceManager): no nat entry found for: 10.151.2.167
file = clients/dpd-client/src/lib.rs:83
|
| 1317 | 2024-10-31T00:14:05.243Z | INFO | SledAgent (ServiceManager): creating nat entry for: 10.151.2.167
file = clients/dpd-client/src/lib.rs:90
|
| 1318 | 2024-10-31T00:14:05.251Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: 10.151.2.164
file = clients/dpd-client/src/lib.rs:118
|
| 1319 | 2024-10-31T00:14:05.253Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1320 | 2024-10-31T00:14:05.291Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: 10.151.2.165
file = clients/dpd-client/src/lib.rs:118
|
| 1321 | 2024-10-31T00:14:05.295Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: 10.151.2.166
file = clients/dpd-client/src/lib.rs:118
|
| 1322 | 2024-10-31T00:14:05.296Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1323 | 2024-10-31T00:14:05.299Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: 10.151.2.168
file = clients/dpd-client/src/lib.rs:118
|
| 1324 | 2024-10-31T00:14:05.299Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1325 | 2024-10-31T00:14:05.302Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1326 | 2024-10-31T00:14:05.302Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: 10.151.2.167
file = clients/dpd-client/src/lib.rs:118
|
| 1327 | 2024-10-31T00:14:05.432Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1328 | 2024-10-31T00:14:05.481Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
file = illumos-utils/src/zone.rs:325
|
| 1329 | 2024-10-31T00:14:05.497Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
file = illumos-utils/src/zone.rs:325
|
| 1330 | 2024-10-31T00:14:05.516Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
file = illumos-utils/src/zone.rs:325
|
| 1331 | 2024-10-31T00:14:05.551Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
file = illumos-utils/src/zone.rs:325
|
| 1332 | 2024-10-31T00:14:05.576Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
file = illumos-utils/src/zone.rs:325
|
| 1333 | 2024-10-31T00:14:05.609Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
file = illumos-utils/src/zone.rs:363
|
| 1334 | 2024-10-31T00:14:05.621Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
file = illumos-utils/src/zone.rs:363
|
| 1335 | 2024-10-31T00:14:05.640Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
file = illumos-utils/src/zone.rs:363
|
| 1336 | 2024-10-31T00:14:05.658Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
file = illumos-utils/src/zone.rs:363
|
| 1337 | 2024-10-31T00:14:05.679Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
file = illumos-utils/src/zone.rs:363
|
| 1338 | 2024-10-31T00:14:08.177Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService13'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::10'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::10'/>
<propval type="astring" name="listen_port" value='17000'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1339 | 2024-10-31T00:14:08.177Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1340 | 2024-10-31T00:14:08.186Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService14'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::f'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::f'/>
<propval type="astring" name="listen_port" value='17000'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1341 | 2024-10-31T00:14:08.192Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService18'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::11'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::11'/>
<propval type="astring" name="listen_port" value='17000'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1342 | 2024-10-31T00:14:08.199Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1343 | 2024-10-31T00:14:08.208Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1344 | 2024-10-31T00:14:08.936Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService23'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::19'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_a536fa1e-3243-42d2-9446-aa31a45af0ee/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::19'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='b60af8cc-0a5c-4e03-bd68-ab85b8054448'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1345 | 2024-10-31T00:14:08.936Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1346 | 2024-10-31T00:14:08.944Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService12'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::14'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_efb62c8a-6541-476c-8447-df5266c7fcb3/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::14'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='fec86277-e836-464f-8814-07b8ff09acb0'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1347 | 2024-10-31T00:14:08.944Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService17'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::17'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::17'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='055300fd-6ed4-449a-b252-4f055b325517'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1348 | 2024-10-31T00:14:08.944Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService10'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::1a'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_b43e5889-7a07-42ec-9599-254e55788daa/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::1a'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='0770a191-23c2-4975-8bf2-3e51b6a0ed52'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1349 | 2024-10-31T00:14:08.944Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1350 | 2024-10-31T00:14:08.951Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1351 | 2024-10-31T00:14:08.959Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1352 | 2024-10-31T00:14:08.966Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService11'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::18'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_a2173f6b-5437-4ef6-a016-ab4b5f89f5d8/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::18'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='06813144-dcfc-43c3-91f0-9fdc37caa5bb'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1353 | 2024-10-31T00:14:08.966Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService15'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::12'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_1d6671a7-5622-4671-b7bf-0a757330cf6b/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::12'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='c3524f4b-5e2f-427e-9689-c5824e0b569f'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1354 | 2024-10-31T00:14:08.966Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1355 | 2024-10-31T00:14:08.973Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1356 | 2024-10-31T00:14:08.980Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService16'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::15'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_ca55523e-99d5-46dd-9043-66e4f721f4aa/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::15'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='66d4d83b-02c1-4519-bb07-849d17436a9b'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1357 | 2024-10-31T00:14:08.981Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService22'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::16'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_67e5554d-50d7-4b6f-b9c3-723d8231ab86/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::16'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='11cdfda5-668b-4581-a6bc-edc534fae5ee'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1358 | 2024-10-31T00:14:08.981Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1359 | 2024-10-31T00:14:08.987Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1360 | 2024-10-31T00:14:09.181Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 50.418459ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1361 | 2024-10-31T00:14:09.194Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 73.562693ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1362 | 2024-10-31T00:14:09.207Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 55.857562ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1363 | 2024-10-31T00:14:09.221Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 29.818995ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1364 | 2024-10-31T00:14:09.232Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 31.551104ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1365 | 2024-10-31T00:14:09.242Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 36.987592ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1366 | 2024-10-31T00:14:09.252Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 39.753867ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1367 | 2024-10-31T00:14:09.264Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 71.984506ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1368 | 2024-10-31T00:14:09.335Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 142.202023ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1369 | 2024-10-31T00:14:09.476Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 73.67873ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1370 | 2024-10-31T00:14:09.628Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 70.310116ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1371 | 2024-10-31T00:14:09.628Z | INFO | SledAgent (ServiceManager): Profile for oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService20'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::d'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/oximeter">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="id" value='4ce79729-70af-4ea9-8907-92eccad979f8'/>
<propval type="astring" name="address" value='[fd00:1122:3344:101::d]:12223'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1372 | 2024-10-31T00:14:09.692Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 60.912733ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1373 | 2024-10-31T00:14:09.692Z | INFO | SledAgent (ServiceManager): Booting oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1374 | 2024-10-31T00:14:09.755Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 103.745635ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1375 | 2024-10-31T00:14:09.793Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 52.699225ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1376 | 2024-10-31T00:14:09.819Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 137.775795ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1377 | 2024-10-31T00:14:09.851Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 57.639006ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1378 | 2024-10-31T00:14:09.881Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 61.235402ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1379 | 2024-10-31T00:14:09.959Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 226.445292ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1380 | 2024-10-31T00:14:10.017Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 73.526448ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1381 | 2024-10-31T00:14:10.091Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 84.672161ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1382 | 2024-10-31T00:14:10.161Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 56.409459ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1383 | 2024-10-31T00:14:10.241Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 76.302432ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1384 | 2024-10-31T00:14:10.242Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService19'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::13'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="dataset" value='oxp_7dd5e852-297b-4341-9eef-72d5112891d5/crucible'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::13'/>
<propval type="astring" name="listen_port" value='32345'/>
<propval type="astring" name="uuid" value='e25e4e7a-bb7a-446a-b584-53e61eb60696'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1385 | 2024-10-31T00:14:10.366Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 137.744494ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1386 | 2024-10-31T00:14:10.413Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 268.761407ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1387 | 2024-10-31T00:14:10.486Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 124.904219ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1388 | 2024-10-31T00:14:10.561Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 234.97103ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1389 | 2024-10-31T00:14:10.606Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 255.359319ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1390 | 2024-10-31T00:14:10.679Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 274.774832ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1391 | 2024-10-31T00:14:10.777Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 357.498718ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1392 | 2024-10-31T00:14:13.790Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 225.935697ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1393 | 2024-10-31T00:14:13.964Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 203.271583ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1394 | 2024-10-31T00:14:14.175Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 124.453297ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1395 | 2024-10-31T00:14:14.248Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 229.533158ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1396 | 2024-10-31T00:14:14.248Z | INFO | SledAgent (ServiceManager): Booting oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1397 | 2024-10-31T00:14:14.312Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 58.629543ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1398 | 2024-10-31T00:14:14.341Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 127.082505ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1399 | 2024-10-31T00:14:14.354Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 429.870256ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1400 | 2024-10-31T00:14:14.366Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 287.406289ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1401 | 2024-10-31T00:14:14.377Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 222.394847ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1402 | 2024-10-31T00:14:14.379Z | INFO | SledAgent (ServiceManager): Profile for oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService24'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::8'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value='opte5'/>
<propval type="astring" name="gateway" value='172.30.1.1'/>
<propval type="astring" name="ip" value='172.30.1.5'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/external_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::8]:5353'/>
<propval type="astring" name="dns_address" value='172.30.1.5:53'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1403 | 2024-10-31T00:14:14.392Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 524.456004ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1404 | 2024-10-31T00:14:14.403Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 395.06363ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1405 | 2024-10-31T00:14:14.413Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 597.051577ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1406 | 2024-10-31T00:14:14.423Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 591.599097ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1407 | 2024-10-31T00:14:14.432Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 498.075697ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1408 | 2024-10-31T00:14:14.442Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 407.496529ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1409 | 2024-10-31T00:14:14.453Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 99.668542ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1410 | 2024-10-31T00:14:14.453Z | INFO | SledAgent (ServiceManager): Booting oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1411 | 2024-10-31T00:14:14.480Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 254.868933ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1412 | 2024-10-31T00:14:14.493Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 398.893328ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1413 | 2024-10-31T00:14:14.569Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 231.812042ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1414 | 2024-10-31T00:14:14.584Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 53.961718ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1415 | 2024-10-31T00:14:14.615Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 474.126007ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1416 | 2024-10-31T00:14:14.615Z | INFO | SledAgent (ServiceManager): Profile for oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService25'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::9'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value='opte1'/>
<propval type="astring" name="gateway" value='172.30.1.1'/>
<propval type="astring" name="ip" value='172.30.1.6'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/external_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::9]:5353'/>
<propval type="astring" name="dns_address" value='172.30.1.6:53'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1417 | 2024-10-31T00:14:14.615Z | INFO | SledAgent (ServiceManager): Booting oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1418 | 2024-10-31T00:14:14.650Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 52.148221ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1419 | 2024-10-31T00:14:14.665Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 988.85805ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1420 | 2024-10-31T00:14:14.691Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 70.201128ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1421 | 2024-10-31T00:14:14.715Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 201.833578ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1422 | 2024-10-31T00:14:14.750Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 682.663998ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1423 | 2024-10-31T00:14:14.772Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 69.052768ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1424 | 2024-10-31T00:14:14.799Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 1.033140241s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1425 | 2024-10-31T00:14:14.813Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 594.883362ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1426 | 2024-10-31T00:14:14.831Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 387.142889ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1427 | 2024-10-31T00:14:14.854Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 114.246521ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1428 | 2024-10-31T00:14:14.865Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 45.071986ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1429 | 2024-10-31T00:14:14.881Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 697.339219ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1430 | 2024-10-31T00:14:14.907Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 957.614216ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1431 | 2024-10-31T00:14:14.923Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 115.578103ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1432 | 2024-10-31T00:14:14.939Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 1.14856388s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1433 | 2024-10-31T00:14:14.959Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 330.785578ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1434 | 2024-10-31T00:14:14.975Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 756.692997ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1435 | 2024-10-31T00:14:14.993Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 201.404526ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1436 | 2024-10-31T00:14:15.026Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 711.279298ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1437 | 2024-10-31T00:14:15.043Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 788.328618ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1438 | 2024-10-31T00:14:15.054Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 177.566563ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1439 | 2024-10-31T00:14:15.103Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 1.372776528s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1440 | 2024-10-31T00:14:15.210Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 1.14028315s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1441 | 2024-10-31T00:14:15.234Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 1.067592399s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1442 | 2024-10-31T00:14:15.248Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 261.135565ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1443 | 2024-10-31T00:14:15.304Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 522.691454ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1444 | 2024-10-31T00:14:15.424Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 668.89078ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1445 | 2024-10-31T00:14:15.428Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService27'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::c'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value='opte3'/>
<propval type="astring" name="gateway" value='172.30.2.1'/>
<propval type="astring" name="ip" value='172.30.2.7'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1446 | 2024-10-31T00:14:15.429Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService26'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::a'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value='opte2'/>
<propval type="astring" name="gateway" value='172.30.2.1'/>
<propval type="astring" name="ip" value='172.30.2.5'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1447 | 2024-10-31T00:14:15.429Z | INFO | SledAgent (ServiceManager): Booting oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1448 | 2024-10-31T00:14:15.448Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 1.258947644s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1449 | 2024-10-31T00:14:15.449Z | INFO | SledAgent (ServiceManager): Booting oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1450 | 2024-10-31T00:14:15.586Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 738.557325ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1451 | 2024-10-31T00:14:15.734Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 816.758556ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1452 | 2024-10-31T00:14:15.842Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 571.239367ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1453 | 2024-10-31T00:14:15.951Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 1.103751518s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1454 | 2024-10-31T00:14:15.990Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 546.415099ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1455 | 2024-10-31T00:14:16.041Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 1.293530589s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1456 | 2024-10-31T00:14:16.108Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 1.047587703s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1457 | 2024-10-31T00:14:16.136Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 1.053997252s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1458 | 2024-10-31T00:14:16.165Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 990.79138ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1459 | 2024-10-31T00:14:16.187Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 571.157648ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1460 | 2024-10-31T00:14:16.216Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 1.498267689s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1461 | 2024-10-31T00:14:16.246Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 37.333501ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1462 | 2024-10-31T00:14:16.263Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 45.512604ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1463 | 2024-10-31T00:14:16.264Z | INFO | SledAgent (ServiceManager): Profile for oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService28'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::b'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value='opte4'/>
<propval type="astring" name="gateway" value='172.30.2.1'/>
<propval type="astring" name="ip" value='172.30.2.6'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/nexus">
<instance enabled="true" name="default">
<property_group type="application" name="config">
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1464 | 2024-10-31T00:14:16.264Z | INFO | SledAgent (ServiceManager): Booting oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1465 | 2024-10-31T00:14:16.298Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 60.978884ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1466 | 2024-10-31T00:14:16.320Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 1.140858641s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1467 | 2024-10-31T00:14:16.333Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 126.319207ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1468 | 2024-10-31T00:14:16.348Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 1.469185944s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1469 | 2024-10-31T00:14:16.368Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 1.267681167s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1470 | 2024-10-31T00:14:16.382Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 255.483164ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1471 | 2024-10-31T00:14:16.430Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 767.336337ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1472 | 2024-10-31T00:14:16.473Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 174.208086ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1473 | 2024-10-31T00:14:16.491Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 789.925924ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1474 | 2024-10-31T00:14:16.552Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 935.957729ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1475 | 2024-10-31T00:14:16.564Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 63.778607ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1476 | 2024-10-31T00:14:16.578Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 1.012986807s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1477 | 2024-10-31T00:14:16.641Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 91.343368ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1478 | 2024-10-31T00:14:16.656Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 500.73849ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1479 | 2024-10-31T00:14:16.671Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 202.596017ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1480 | 2024-10-31T00:14:16.721Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 877.165304ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1481 | 2024-10-31T00:14:16.746Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 299.997512ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1482 | 2024-10-31T00:14:16.772Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 978.781361ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1483 | 2024-10-31T00:14:16.891Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 1.014601741s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1484 | 2024-10-31T00:14:17.064Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 547.48047ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1485 | 2024-10-31T00:14:17.078Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 1.34733082s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1486 | 2024-10-31T00:14:17.171Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 1.234736159s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1487 | 2024-10-31T00:14:17.186Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 670.138386ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1488 | 2024-10-31T00:14:17.201Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 1.405421037s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1489 | 2024-10-31T00:14:17.217Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 1.087796086s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1490 | 2024-10-31T00:14:17.235Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 596.762579ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1491 | 2024-10-31T00:14:17.298Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 1.4852283s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1492 | 2024-10-31T00:14:17.350Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 779.158606ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1493 | 2024-10-31T00:14:17.483Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 1.33946721s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1494 | 2024-10-31T00:14:17.511Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 724.833704ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1495 | 2024-10-31T00:14:17.622Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 1.128459107s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1496 | 2024-10-31T00:14:17.643Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 1.177217866s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1497 | 2024-10-31T00:14:17.667Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 897.051821ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1498 | 2024-10-31T00:14:17.710Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 1.457310909s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1499 | 2024-10-31T00:14:17.742Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 1.267254854s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1500 | 2024-10-31T00:14:17.768Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 1.346399303s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1501 | 2024-10-31T00:14:17.836Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 1.22929114s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1502 | 2024-10-31T00:14:17.853Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 723.531133ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1503 | 2024-10-31T00:14:17.874Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 1.450168767s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1504 | 2024-10-31T00:14:17.941Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 1.299051545s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1505 | 2024-10-31T00:14:18.148Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 576.399926ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1506 | 2024-10-31T00:14:18.249Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 1.462388573s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1507 | 2024-10-31T00:14:18.272Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 2.45981675s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:424
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 1508 | 2024-10-31T00:14:18.318Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 666.558845ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1509 | 2024-10-31T00:14:18.420Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 917.915011ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1510 | 2024-10-31T00:14:18.439Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 840.972427ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1511 | 2024-10-31T00:14:18.440Z | INFO | SledAgent (ServiceManager): Profile for oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService21'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::e'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/clickhouse">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::e'/>
<propval type="astring" name="listen_port" value='8123'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/clickhouse-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::e]:8888'/>
<propval type="astring" name="ch_address" value='[fd00:1122:3344:101::e]:8123'/>
<propval type="astring" name="ch_binary" value='/opt/oxide/clickhouse/clickhouse'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1512 | 2024-10-31T00:14:18.440Z | INFO | SledAgent (ServiceManager): Booting oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1513 | 2024-10-31T00:14:18.584Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 652.821293ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1514 | 2024-10-31T00:14:18.599Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 1.377586128s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1515 | 2024-10-31T00:14:18.622Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 611.016369ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1516 | 2024-10-31T00:14:18.748Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 733.195866ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1517 | 2024-10-31T00:14:18.801Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 1.476704184s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1518 | 2024-10-31T00:14:18.827Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 931.956881ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1519 | 2024-10-31T00:14:18.842Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 47.188772ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1520 | 2024-10-31T00:14:18.863Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 826.770439ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1521 | 2024-10-31T00:14:18.907Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 994.107873ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1522 | 2024-10-31T00:14:18.922Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 123.825066ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1523 | 2024-10-31T00:14:19.023Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 1.378127885s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1524 | 2024-10-31T00:14:19.065Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 661.687454ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1525 | 2024-10-31T00:14:19.105Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 208.41022ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1526 | 2024-10-31T00:14:19.132Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 643.063922ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1527 | 2024-10-31T00:14:19.153Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 998.83149ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1528 | 2024-10-31T00:14:19.182Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 954.660276ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1529 | 2024-10-31T00:14:19.247Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 866.505265ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1530 | 2024-10-31T00:14:19.260Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 1.176410284s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1531 | 2024-10-31T00:14:19.274Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 1.429742991s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1532 | 2024-10-31T00:14:19.291Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 1.443761617s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1533 | 2024-10-31T00:14:19.328Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 545.904948ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1534 | 2024-10-31T00:14:19.339Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 897.084244ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1535 | 2024-10-31T00:14:19.351Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 1.307643954s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1536 | 2024-10-31T00:14:19.500Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 934.512535ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1537 | 2024-10-31T00:14:19.711Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 948.634448ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1538 | 2024-10-31T00:14:19.727Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 1.137135849s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1539 | 2024-10-31T00:14:19.743Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 1.055457756s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1540 | 2024-10-31T00:14:19.772Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 806.04501ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1541 | 2024-10-31T00:14:19.792Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 680.337685ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1542 | 2024-10-31T00:14:19.894Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 1.17194932s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1543 | 2024-10-31T00:14:19.918Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 649.509436ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1544 | 2024-10-31T00:14:19.993Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 1.300707068s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1545 | 2024-10-31T00:14:20.131Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 1.407183567s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1546 | 2024-10-31T00:14:20.154Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 1.393530484s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1547 | 2024-10-31T00:14:20.168Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 805.128605ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1548 | 2024-10-31T00:14:20.249Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 921.539819ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1549 | 2024-10-31T00:14:20.290Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 1.125650062s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1550 | 2024-10-31T00:14:20.415Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 1.027437774s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1551 | 2024-10-31T00:14:20.445Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 1.418098447s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1552 | 2024-10-31T00:14:20.454Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 1.043610961s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1553 | 2024-10-31T00:14:20.484Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 818.104284ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1554 | 2024-10-31T00:14:20.581Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 1.421725341s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1555 | 2024-10-31T00:14:20.594Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 1.480323268s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1556 | 2024-10-31T00:14:20.673Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 627.058428ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1557 | 2024-10-31T00:14:20.685Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 903.813406ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1558 | 2024-10-31T00:14:20.717Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 710.255714ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1559 | 2024-10-31T00:14:20.824Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 511.113412ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1560 | 2024-10-31T00:14:20.837Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 872.833974ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1561 | 2024-10-31T00:14:20.880Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 773.406698ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1562 | 2024-10-31T00:14:20.987Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 1.497833535s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1563 | 2024-10-31T00:14:21.080Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 1.238925723s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1564 | 2024-10-31T00:14:21.186Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 706.539734ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1565 | 2024-10-31T00:14:21.308Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 1.274459183s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1566 | 2024-10-31T00:14:21.321Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 1.031157681s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1567 | 2024-10-31T00:14:21.331Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 1.07394376s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1568 | 2024-10-31T00:14:21.346Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 1.280596559s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1569 | 2024-10-31T00:14:21.430Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 1.023144697s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1570 | 2024-10-31T00:14:21.444Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 1.013270914s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1571 | 2024-10-31T00:14:21.455Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 808.032351ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1572 | 2024-10-31T00:14:21.510Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 1.377358747s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1573 | 2024-10-31T00:14:21.552Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 568.216725ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1574 | 2024-10-31T00:14:21.562Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 1.369846251s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1575 | 2024-10-31T00:14:21.600Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 1.277891518s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1576 | 2024-10-31T00:14:21.665Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 836.414693ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1577 | 2024-10-31T00:14:21.720Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 884.105795ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1578 | 2024-10-31T00:14:21.877Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 1.173694136s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1579 | 2024-10-31T00:14:21.906Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 1.205345634s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1580 | 2024-10-31T00:14:22.015Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 1.396279738s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1581 | 2024-10-31T00:14:22.086Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 1.327345087s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1582 | 2024-10-31T00:14:22.133Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 928.261978ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1583 | 2024-10-31T00:14:22.276Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 1.411675147s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1584 | 2024-10-31T00:14:22.329Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 981.952023ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1585 | 2024-10-31T00:14:22.362Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 1.398416421s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1586 | 2024-10-31T00:14:22.416Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 686.204358ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1587 | 2024-10-31T00:14:22.466Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 536.606537ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1588 | 2024-10-31T00:14:22.475Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 814.89308ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1589 | 2024-10-31T00:14:22.495Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 822.486946ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1590 | 2024-10-31T00:14:22.514Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 1.254131965s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1591 | 2024-10-31T00:14:22.591Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 801.905924ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1592 | 2024-10-31T00:14:22.615Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 1.156271368s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1593 | 2024-10-31T00:14:22.638Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 524.612066ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1594 | 2024-10-31T00:14:22.891Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 823.936967ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1595 | 2024-10-31T00:14:22.901Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 1.355339161s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1596 | 2024-10-31T00:14:22.943Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 976.452959ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1597 | 2024-10-31T00:14:23.016Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 1.06376959s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1598 | 2024-10-31T00:14:23.062Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 1.48006584s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1599 | 2024-10-31T00:14:23.074Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 786.148242ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1600 | 2024-10-31T00:14:23.115Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 1.411149617s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1601 | 2024-10-31T00:14:23.126Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 1.250522546s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1602 | 2024-10-31T00:14:23.177Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 1.225582687s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1603 | 2024-10-31T00:14:23.303Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 1.02936123s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1604 | 2024-10-31T00:14:23.326Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 1.324801375s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1605 | 2024-10-31T00:14:23.338Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 600.335679ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1606 | 2024-10-31T00:14:23.406Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 1.08655399s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1607 | 2024-10-31T00:14:23.423Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 1.266592864s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1608 | 2024-10-31T00:14:23.433Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 756.675059ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1609 | 2024-10-31T00:14:23.702Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 1.265574955s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1610 | 2024-10-31T00:14:23.727Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 1.204979336s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1611 | 2024-10-31T00:14:23.779Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 1.324455532s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1612 | 2024-10-31T00:14:23.792Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 908.906563ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1613 | 2024-10-31T00:14:23.804Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 958.792023ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1614 | 2024-10-31T00:14:23.874Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 588.925453ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1615 | 2024-10-31T00:14:23.932Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 760.066782ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1616 | 2024-10-31T00:14:23.949Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 1.000775818s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1617 | 2024-10-31T00:14:24.094Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 1.042803125s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1618 | 2024-10-31T00:14:24.228Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 1.376467428s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1619 | 2024-10-31T00:14:24.270Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 1.127285591s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1620 | 2024-10-31T00:14:24.349Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 610.98408ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1621 | 2024-10-31T00:14:24.391Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 710.19471ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1622 | 2024-10-31T00:14:24.414Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 1.173637111s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1623 | 2024-10-31T00:14:24.476Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 1.312561224s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1624 | 2024-10-31T00:14:24.505Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 1.040510325s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1625 | 2024-10-31T00:14:24.538Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 810.865116ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1626 | 2024-10-31T00:14:24.555Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 732.183773ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1627 | 2024-10-31T00:14:24.663Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 1.324346852s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1628 | 2024-10-31T00:14:24.703Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 516.827752ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1629 | 2024-10-31T00:14:24.714Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 753.802438ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1630 | 2024-10-31T00:14:24.726Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 573.229231ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1631 | 2024-10-31T00:14:24.776Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 1.358402986s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1632 | 2024-10-31T00:14:24.945Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 720.928826ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1633 | 2024-10-31T00:14:24.963Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 945.253083ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1634 | 2024-10-31T00:14:24.973Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 712.655742ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1635 | 2024-10-31T00:14:24.982Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 1.101409159s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1636 | 2024-10-31T00:14:25.113Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 745.871733ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1637 | 2024-10-31T00:14:25.123Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 1.106050525s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1638 | 2024-10-31T00:14:25.147Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 1.475264945s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1639 | 2024-10-31T00:14:25.230Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 854.124381ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1640 | 2024-10-31T00:14:25.298Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 1.102269443s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1641 | 2024-10-31T00:14:25.309Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 1.186336032s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1642 | 2024-10-31T00:14:25.360Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 699.757502ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1643 | 2024-10-31T00:14:25.408Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 1.090339461s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1644 | 2024-10-31T00:14:25.478Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 600.711076ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1645 | 2024-10-31T00:14:25.556Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 534.972134ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1646 | 2024-10-31T00:14:25.600Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 1.176481721s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1647 | 2024-10-31T00:14:25.615Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 558.967999ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1648 | 2024-10-31T00:14:25.677Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 719.975328ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1649 | 2024-10-31T00:14:25.696Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 863.40711ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1650 | 2024-10-31T00:14:25.801Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 1.285680545s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1651 | 2024-10-31T00:14:25.870Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 708.267822ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1652 | 2024-10-31T00:14:25.920Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 1.388999235s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1653 | 2024-10-31T00:14:26.000Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 1.30003677s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1654 | 2024-10-31T00:14:26.070Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 1.021616773s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1655 | 2024-10-31T00:14:26.089Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 861.662985ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1656 | 2024-10-31T00:14:26.098Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 785.815392ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1657 | 2024-10-31T00:14:26.107Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 1.356218231s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1658 | 2024-10-31T00:14:26.115Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 1.212315518s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1659 | 2024-10-31T00:14:26.144Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 1.253765165s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1660 | 2024-10-31T00:14:26.184Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 840.314639ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1661 | 2024-10-31T00:14:26.239Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 1.357798397s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1662 | 2024-10-31T00:14:26.409Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 1.37978529s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1663 | 2024-10-31T00:14:26.417Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 604.947544ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1664 | 2024-10-31T00:14:26.506Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 1.184582143s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1665 | 2024-10-31T00:14:26.514Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 1.133158222s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1666 | 2024-10-31T00:14:26.569Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 1.211205292s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1667 | 2024-10-31T00:14:26.588Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 705.256415ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1668 | 2024-10-31T00:14:26.631Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 1.286772023s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1669 | 2024-10-31T00:14:26.788Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 1.079471004s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1670 | 2024-10-31T00:14:26.893Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 1.379929496s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1671 | 2024-10-31T00:14:26.960Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 1.365166217s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1672 | 2024-10-31T00:14:27.031Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 675.811001ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1673 | 2024-10-31T00:14:27.039Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 567.555737ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1674 | 2024-10-31T00:14:27.095Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 897.540302ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1675 | 2024-10-31T00:14:27.108Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 1.108579134s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1676 | 2024-10-31T00:14:27.302Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 1.167367235s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1677 | 2024-10-31T00:14:27.310Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 1.420239723s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1678 | 2024-10-31T00:14:27.318Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 529.708781ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1679 | 2024-10-31T00:14:27.336Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 805.304598ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1680 | 2024-10-31T00:14:27.408Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 620.642605ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1681 | 2024-10-31T00:14:27.473Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 1.043642569s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1682 | 2024-10-31T00:14:27.606Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 1.380680489s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1683 | 2024-10-31T00:14:27.617Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 539.059716ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1684 | 2024-10-31T00:14:27.657Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 710.971345ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1685 | 2024-10-31T00:14:27.699Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 1.153230358s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1686 | 2024-10-31T00:14:27.715Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 552.225076ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1687 | 2024-10-31T00:14:27.788Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 852.268996ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1688 | 2024-10-31T00:14:27.795Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 706.664997ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1689 | 2024-10-31T00:14:27.857Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 782.288969ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1690 | 2024-10-31T00:14:27.876Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 724.575015ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1691 | 2024-10-31T00:14:27.928Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 533.038658ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1692 | 2024-10-31T00:14:28.001Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 1.024023558s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1693 | 2024-10-31T00:14:28.040Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 1.000116444s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1694 | 2024-10-31T00:14:28.150Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 708.56572ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1695 | 2024-10-31T00:14:28.164Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 1.258501167s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1696 | 2024-10-31T00:14:28.226Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 1.136906554s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1697 | 2024-10-31T00:14:28.277Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 1.490480316s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1698 | 2024-10-31T00:14:28.285Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 1.282093026s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1699 | 2024-10-31T00:14:28.334Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 1.345277898s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1700 | 2024-10-31T00:14:28.377Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 718.476623ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1701 | 2024-10-31T00:14:28.471Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 684.720834ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1702 | 2024-10-31T00:14:28.478Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 598.429609ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1703 | 2024-10-31T00:14:28.511Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 633.489234ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1704 | 2024-10-31T00:14:28.524Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 764.313209ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1705 | 2024-10-31T00:14:28.609Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 1.031034942s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1706 | 2024-10-31T00:14:28.647Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 1.255977186s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1707 | 2024-10-31T00:14:28.656Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 831.859633ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1708 | 2024-10-31T00:14:28.740Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 729.684289ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1709 | 2024-10-31T00:14:28.861Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 1.13329589s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1710 | 2024-10-31T00:14:28.869Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a") failed: Property not found. retry in 1.222462112s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1711 | 2024-10-31T00:14:28.996Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 877.391679ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1712 | 2024-10-31T00:14:29.035Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 1.176124843s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1713 | 2024-10-31T00:14:29.049Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 648.236903ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1714 | 2024-10-31T00:14:29.086Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 680.752248ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1715 | 2024-10-31T00:14:29.104Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 1.295774784s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1716 | 2024-10-31T00:14:29.153Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 565.788616ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1717 | 2024-10-31T00:14:29.163Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 863.897073ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1718 | 2024-10-31T00:14:29.298Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 967.72028ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1719 | 2024-10-31T00:14:29.371Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87") failed: Property not found. retry in 1.145631407s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1720 | 2024-10-31T00:14:29.432Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b") failed: Property not found. retry in 641.003484ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1721 | 2024-10-31T00:14:29.479Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 947.995974ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1722 | 2024-10-31T00:14:29.497Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 1.059166348s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1723 | 2024-10-31T00:14:29.576Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c") failed: Property not found. retry in 1.474053408s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1724 | 2024-10-31T00:14:29.647Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c") failed: Property not found. retry in 745.8462ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1725 | 2024-10-31T00:14:29.688Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 952.070931ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1726 | 2024-10-31T00:14:29.706Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586") failed: Property not found. retry in 1.462480157s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1727 | 2024-10-31T00:14:29.726Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 1.319079946s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1728 | 2024-10-31T00:14:29.774Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 1.004951238s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1729 | 2024-10-31T00:14:29.782Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738") failed: Property not found. retry in 1.011763786s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1730 | 2024-10-31T00:14:29.883Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7") failed: Property not found. retry in 1.460482158s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1731 | 2024-10-31T00:14:29.913Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4") failed: Property not found. retry in 1.352561976s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1732 | 2024-10-31T00:14:30.004Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb") failed: Property not found. retry in 759.284546ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1733 | 2024-10-31T00:14:30.037Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386") failed: Property not found. retry in 1.05974926s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1734 | 2024-10-31T00:14:30.224Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7") failed: Property not found. retry in 618.221612ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1735 | 2024-10-31T00:14:30.277Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8") failed: Property not found. retry in 868.232574ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1736 | 2024-10-31T00:14:30.418Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7") failed: Property not found. retry in 648.164554ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1737 | 2024-10-31T00:14:30.437Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 540.376715ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1738 | 2024-10-31T00:14:30.567Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86") failed: Property not found. retry in 940.12013ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1739 | 2024-10-31T00:14:30.627Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:49845
|
| 1740 | 2024-10-31T00:14:30.629Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 659
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:49845
req_id = 84fd9ce4-ad6e-4898-9076-5bf842869a62
response_code = 200
uri = /baseboard
|
| 1741 | 2024-10-31T00:14:30.650Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988") failed: Property not found. retry in 849.66252ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1742 | 2024-10-31T00:14:30.789Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 990.090577ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1743 | 2024-10-31T00:14:31.221Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3") failed: Property not found. retry in 1.328759942s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1744 | 2024-10-31T00:14:31.402Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 560.444296ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1745 | 2024-10-31T00:14:31.791Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c") failed: Property not found. retry in 776.472748ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1746 | 2024-10-31T00:14:31.972Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 703.110377ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1747 | 2024-10-31T00:14:32.694Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 518.113745ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1748 | 2024-10-31T00:14:33.231Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 1.436111904s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1749 | 2024-10-31T00:14:34.692Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 1.241687799s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1750 | 2024-10-31T00:14:35.812Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 3.142985321s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:424
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 1751 | 2024-10-31T00:14:35.949Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 1.070552566s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1752 | 2024-10-31T00:14:37.029Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 843.518088ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1753 | 2024-10-31T00:14:37.883Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 1.424848507s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1754 | 2024-10-31T00:14:38.990Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 5.305553412s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:424
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 1755 | 2024-10-31T00:14:39.320Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 786.306849ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1756 | 2024-10-31T00:14:40.116Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 858.830246ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1757 | 2024-10-31T00:14:40.984Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 558.35586ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1758 | 2024-10-31T00:14:41.554Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 987.557765ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1759 | 2024-10-31T00:14:42.552Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 865.256396ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1760 | 2024-10-31T00:14:43.427Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 810.906019ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1761 | 2024-10-31T00:14:44.249Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 1.152796026s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1762 | 2024-10-31T00:14:44.331Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 18.210721394s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:424
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 1763 | 2024-10-31T00:14:45.413Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 1.045169045s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1764 | 2024-10-31T00:14:46.470Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f") failed: Property not found. retry in 776.069626ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1765 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3
|
| 1766 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c
|
| 1767 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b
|
| 1768 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7
|
| 1769 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4
|
| 1770 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c
|
| 1771 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988
|
| 1772 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586
|
| 1773 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738
|
| 1774 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7
|
| 1775 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386
|
| 1776 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87
|
| 1777 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a
|
| 1778 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8
|
| 1779 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86
|
| 1780 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c
|
| 1781 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f
|
| 1782 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb
|
| 1783 | 2024-10-31T00:14:47.285Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3273
zone = oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7
|
| 1784 | 2024-10-31T00:14:47.286Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 1785 | 2024-10-31T00:14:47.290Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 1786 | 2024-10-31T00:14:47.292Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 45029165
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:44733
req_id = 1d8fad99-c34b-416c-aa53-c17fe306dd9b
response_code = 204
uri = /omicron-zones
|
| 1787 | 2024-10-31T00:14:47.299Z | INFO | SledAgent (RSS): Finished setting up services
file = sled-agent/src/rack_setup/service.rs:1322
|
| 1788 | 2024-10-31T00:14:47.300Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.rss-plan-completed.marker.tmp
file = common/src/ledger.rs:199
|
| 1789 | 2024-10-31T00:14:47.300Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.rss-plan-completed.marker.tmp
file = common/src/ledger.rs:199
|
| 1790 | 2024-10-31T00:14:47.308Z | INFO | SledAgent (RSS): Handing off control to Nexus
file = sled-agent/src/rack_setup/service.rs:708
|
| 1791 | 2024-10-31T00:14:47.310Z | INFO | SledAgent (RSS): Nexus address: [fd00:1122:3344:101::a]:12221
file = sled-agent/src/rack_setup/service.rs:720
|
| 1792 | 2024-10-31T00:14:47.347Z | INFO | SledAgent (RSS): rack_network_config: RackNetworkConfigV2 {
bfd: [],
bgp: [],
infra_ip_first: 10.151.2.174,
infra_ip_last: 10.151.2.174,
ports: [
PortConfigV2 {
addresses: [
UplinkAddressConfig {
address: V4(
Ipv4Net {
addr: 10.151.2.174,
width: 24,
},
),
vlan_id: None,
},
],
autoneg: false,
bgp_peers: [],
lldp: None,
port: "qsfp0",
routes: [
RouteConfig {
destination: V4(
Ipv4Net {
addr: 0.0.0.0,
width: 0,
},
),
nexthop: 10.151.2.100,
rib_priority: None,
vlan_id: None,
},
],
switch: Switch0,
uplink_port_fec: None,
uplink_port_speed: Speed40G,
},
],
rack_subnet: Ipv6Net {
addr: fd00:1122:3344:100::,
width: 56,
},
}
file = sled-agent/src/rack_setup/service.rs:887
|
| 1793 | 2024-10-31T00:14:48.475Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "bf6bc666-c3c5-4b42-bff6-051525a8f889", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:48 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "bf6bc666-c3c5-4b42-bff6-051525a8f889" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1794 | 2024-10-31T00:14:49.178Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "e1514bc0-b5c6-468a-8e33-12fec3c672ff", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:49 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "e1514bc0-b5c6-468a-8e33-12fec3c672ff" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1795 | 2024-10-31T00:14:49.602Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "4b070543-185e-4c5a-93cc-b9910e6c4430", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:49 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "4b070543-185e-4c5a-93cc-b9910e6c4430" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1796 | 2024-10-31T00:14:50.208Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "470d967a-09e2-4d3e-a3cd-b69922caca2d", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:50 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "470d967a-09e2-4d3e-a3cd-b69922caca2d" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1797 | 2024-10-31T00:14:50.887Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "7bef6f07-0f85-4228-ad68-0e541cfda12a", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:50 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "7bef6f07-0f85-4228-ad68-0e541cfda12a" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1798 | 2024-10-31T00:14:51.303Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "99de20f9-5c3a-4798-aa66-17091a6089ab", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:51 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "99de20f9-5c3a-4798-aa66-17091a6089ab" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1799 | 2024-10-31T00:14:51.827Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "c8f86926-3263-4faa-a216-ff5e9d1c6710", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:51 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "c8f86926-3263-4faa-a216-ff5e9d1c6710" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1800 | 2024-10-31T00:14:52.414Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "d62a61f3-7640-4511-8813-f5668d4f72c4", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:51 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "d62a61f3-7640-4511-8813-f5668d4f72c4" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1801 | 2024-10-31T00:14:53.049Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "88269fac-003c-4ff7-93d6-d585b92beb37", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:52 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "88269fac-003c-4ff7-93d6-d585b92beb37" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1802 | 2024-10-31T00:14:53.637Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "d9f5d99e-d559-4852-b9b6-f24aff0f26b5", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:53 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "d9f5d99e-d559-4852-b9b6-f24aff0f26b5" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1803 | 2024-10-31T00:14:54.623Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "b704df44-738f-49e3-ba47-a3d4195a8f1c", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:54 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "b704df44-738f-49e3-ba47-a3d4195a8f1c" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1804 | 2024-10-31T00:14:55.551Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "20985e1c-2dce-4051-8356-69e5daeebae7", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:55 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "20985e1c-2dce-4051-8356-69e5daeebae7" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1805 | 2024-10-31T00:14:56.439Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "c819ea48-aa7e-4d5e-8c31-7a7ebf55a304", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:55 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "c819ea48-aa7e-4d5e-8c31-7a7ebf55a304" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1806 | 2024-10-31T00:14:57.311Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "cc3db658-e74a-490c-b719-bbf1ccc273e0", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:57 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "cc3db658-e74a-490c-b719-bbf1ccc273e0" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1807 | 2024-10-31T00:14:58.148Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "974919b2-b629-4c01-8c0f-b0b1b61453e3", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:57 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "974919b2-b629-4c01-8c0f-b0b1b61453e3" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1808 | 2024-10-31T00:14:59.166Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "a5d079b1-ae1d-4f77-baa8-0e5a1d647217", "content-length": "173", "date": "Thu, 31 Oct 2024 00:14:59 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "a5d079b1-ae1d-4f77-baa8-0e5a1d647217" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1809 | 2024-10-31T00:15:00.673Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:60820
|
| 1810 | 2024-10-31T00:15:00.673Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 32
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:60820
req_id = 82940038-c41d-45b7-9547-9e9458a7f880
response_code = 200
uri = /baseboard
|
| 1811 | 2024-10-31T00:15:01.002Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "f0b25084-3d14-4a4c-aa61-4aa4cbfd67d3", "content-length": "173", "date": "Thu, 31 Oct 2024 00:15:00 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "f0b25084-3d14-4a4c-aa61-4aa4cbfd67d3" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1812 | 2024-10-31T00:15:03.181Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "29555811-8b17-4459-97bd-cf2014c4e5fd", "content-length": "173", "date": "Thu, 31 Oct 2024 00:15:03 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "29555811-8b17-4459-97bd-cf2014c4e5fd" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1813 | 2024-10-31T00:15:04.854Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "972b28bc-5e7d-4f60-ac6d-0860660d783a", "content-length": "173", "date": "Thu, 31 Oct 2024 00:15:04 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: sled with id \"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\"", request_id: "972b28bc-5e7d-4f60-ac6d-0860660d783a" }
file = sled-agent/src/rack_setup/service.rs:949
|
| 1814 | 2024-10-31T00:15:05.579Z | INFO | SledAgent (NexusNotifierTask): Successfully put SledAgentInfo to nexus
file = sled-agent/src/nexus.rs:500
new_generation = 1
old_generation = 1
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 1815 | 2024-10-31T00:15:05.678Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:39169
|
| 1816 | 2024-10-31T00:15:05.680Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1817 | 2024-10-31T00:15:05.680Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1818 | 2024-10-31T00:15:05.681Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1819 | 2024-10-31T00:15:05.682Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1820 | 2024-10-31T00:15:05.867Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 188523
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:39169
req_id = a5ea8669-f19c-4a2c-816a-c5d4ddde47c2
response_code = 200
uri = /inventory
|
| 1821 | 2024-10-31T00:15:08.082Z | INFO | SledAgent (dropshot): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:40997
remote_addr = [fd00:1122:3344:101::d]:50335
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 1822 | 2024-10-31T00:15:08.109Z | INFO | SledAgent (dropshot): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 25234
local_addr = [fd00:1122:3344:101::1]:40997
method = GET
remote_addr = [fd00:1122:3344:101::d]:50335
req_id = a57162a2-f34c-4141-ab78-0e09b2cb70d8
response_code = 200
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
uri = /f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 1823 | 2024-10-31T00:15:09.183Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:47812
|
| 1824 | 2024-10-31T00:15:09.189Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:731
rules = [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea2ccb0-4d8d-450a-b5fa-20f11d99c6b4, kind: Service { id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7 }, name: Name("external-dns-7ed51b56-e87d-4a9a-b9bc-27ece36a86d7"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 486dd504-bd2f-498e-8990-b1d7d833ab10, kind: Service { id: b2beb08a-ae3f-469f-86c4-36645336e7c7 }, name: Name("external-dns-b2beb08a-ae3f-469f-86c4-36645336e7c7"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 100])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1e6d66a8-381f-4d1d-a719-7f1f1a2fab2f, kind: Service { id: 1e4bfc38-893c-408c-9d22-948390ba9d86 }, name: Name("nexus-1e4bfc38-893c-408c-9d22-948390ba9d86"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 239, 133])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 397cb5ae-79f3-4840-95d1-a308d95a2fad, kind: Service { id: 2c60990c-614d-44ea-a7bf-2bae04edc1b3 }, name: Name("nexus-2c60990c-614d-44ea-a7bf-2bae04edc1b3"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 222])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f546a11c-0e59-4819-8b6f-c01e0cfa393a, kind: Service { id: 1f4e8f94-7bc7-4b29-b92c-ee717847f51c }, name: Name("nexus-1f4e8f94-7bc7-4b29-b92c-ee717847f51c"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 1825 | 2024-10-31T00:15:09.189Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 1826 | 2024-10-31T00:15:09.189Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 1827 | 2024-10-31T00:15:09.189Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte0"
rules = []
|
| 1828 | 2024-10-31T00:15:09.189Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 1829 | 2024-10-31T00:15:09.189Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 1830 | 2024-10-31T00:15:09.189Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 1831 | 2024-10-31T00:15:09.189Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 5830
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:47812
req_id = ac5ccf05-c15c-4abb-bf89-89961eeb0e47
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 1832 | 2024-10-31T00:15:09.191Z | INFO | SledAgent (RSS): Handoff to Nexus is complete
file = sled-agent/src/rack_setup/service.rs:959
|
| 1833 | 2024-10-31T00:15:09.449Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:40258
|
| 1834 | 2024-10-31T00:15:09.449Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1835 | 2024-10-31T00:15:09.449Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 1836 | 2024-10-31T00:15:09.449Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1837 | 2024-10-31T00:15:09.450Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 1838 | 2024-10-31T00:15:09.641Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 191548
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:40258
req_id = 181665ad-e73d-456a-8aad-fc995a31a6cf
response_code = 200
uri = /inventory
|
| 1839 | 2024-10-31T00:15:10.786Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:56715
|
| 1840 | 2024-10-31T00:15:10.787Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:731
rules = [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea2ccb0-4d8d-450a-b5fa-20f11d99c6b4, kind: Service { id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7 }, name: Name("external-dns-7ed51b56-e87d-4a9a-b9bc-27ece36a86d7"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 486dd504-bd2f-498e-8990-b1d7d833ab10, kind: Service { id: b2beb08a-ae3f-469f-86c4-36645336e7c7 }, name: Name("external-dns-b2beb08a-ae3f-469f-86c4-36645336e7c7"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 100])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1e6d66a8-381f-4d1d-a719-7f1f1a2fab2f, kind: Service { id: 1e4bfc38-893c-408c-9d22-948390ba9d86 }, name: Name("nexus-1e4bfc38-893c-408c-9d22-948390ba9d86"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 239, 133])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 397cb5ae-79f3-4840-95d1-a308d95a2fad, kind: Service { id: 2c60990c-614d-44ea-a7bf-2bae04edc1b3 }, name: Name("nexus-2c60990c-614d-44ea-a7bf-2bae04edc1b3"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 222])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f546a11c-0e59-4819-8b6f-c01e0cfa393a, kind: Service { id: 1f4e8f94-7bc7-4b29-b92c-ee717847f51c }, name: Name("nexus-1f4e8f94-7bc7-4b29-b92c-ee717847f51c"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 1841 | 2024-10-31T00:15:10.787Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 1842 | 2024-10-31T00:15:10.787Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 1843 | 2024-10-31T00:15:10.787Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte0"
rules = []
|
| 1844 | 2024-10-31T00:15:10.787Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 1845 | 2024-10-31T00:15:10.787Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 1846 | 2024-10-31T00:15:10.787Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 1847 | 2024-10-31T00:15:10.787Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 330
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:56715
req_id = 9f941524-701e-4e52-be50-a6bac6d1decd
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 1848 | 2024-10-31T00:15:10.795Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:57926
|
| 1849 | 2024-10-31T00:15:10.795Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:731
rules = [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea2ccb0-4d8d-450a-b5fa-20f11d99c6b4, kind: Service { id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7 }, name: Name("external-dns-7ed51b56-e87d-4a9a-b9bc-27ece36a86d7"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 486dd504-bd2f-498e-8990-b1d7d833ab10, kind: Service { id: b2beb08a-ae3f-469f-86c4-36645336e7c7 }, name: Name("external-dns-b2beb08a-ae3f-469f-86c4-36645336e7c7"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 100])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1e6d66a8-381f-4d1d-a719-7f1f1a2fab2f, kind: Service { id: 1e4bfc38-893c-408c-9d22-948390ba9d86 }, name: Name("nexus-1e4bfc38-893c-408c-9d22-948390ba9d86"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 239, 133])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 397cb5ae-79f3-4840-95d1-a308d95a2fad, kind: Service { id: 2c60990c-614d-44ea-a7bf-2bae04edc1b3 }, name: Name("nexus-2c60990c-614d-44ea-a7bf-2bae04edc1b3"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 222])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f546a11c-0e59-4819-8b6f-c01e0cfa393a, kind: Service { id: 1f4e8f94-7bc7-4b29-b92c-ee717847f51c }, name: Name("nexus-1f4e8f94-7bc7-4b29-b92c-ee717847f51c"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 1850 | 2024-10-31T00:15:10.795Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 1851 | 2024-10-31T00:15:10.795Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 1852 | 2024-10-31T00:15:10.795Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte0"
rules = []
|
| 1853 | 2024-10-31T00:15:10.795Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 1854 | 2024-10-31T00:15:10.795Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 1855 | 2024-10-31T00:15:10.795Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 1856 | 2024-10-31T00:15:10.795Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 246
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:57926
req_id = 2b53a1c8-d913-4ac7-9af6-34f11659a1c1
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 1857 | 2024-10-31T00:15:10.847Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:40636
|
| 1858 | 2024-10-31T00:15:10.848Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:731
rules = [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea2ccb0-4d8d-450a-b5fa-20f11d99c6b4, kind: Service { id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7 }, name: Name("external-dns-7ed51b56-e87d-4a9a-b9bc-27ece36a86d7"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 486dd504-bd2f-498e-8990-b1d7d833ab10, kind: Service { id: b2beb08a-ae3f-469f-86c4-36645336e7c7 }, name: Name("external-dns-b2beb08a-ae3f-469f-86c4-36645336e7c7"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 100])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1e6d66a8-381f-4d1d-a719-7f1f1a2fab2f, kind: Service { id: 1e4bfc38-893c-408c-9d22-948390ba9d86 }, name: Name("nexus-1e4bfc38-893c-408c-9d22-948390ba9d86"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 239, 133])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 397cb5ae-79f3-4840-95d1-a308d95a2fad, kind: Service { id: 2c60990c-614d-44ea-a7bf-2bae04edc1b3 }, name: Name("nexus-2c60990c-614d-44ea-a7bf-2bae04edc1b3"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 222])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f546a11c-0e59-4819-8b6f-c01e0cfa393a, kind: Service { id: 1f4e8f94-7bc7-4b29-b92c-ee717847f51c }, name: Name("nexus-1f4e8f94-7bc7-4b29-b92c-ee717847f51c"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 1859 | 2024-10-31T00:15:10.848Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 1860 | 2024-10-31T00:15:10.848Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 1861 | 2024-10-31T00:15:10.848Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte0"
rules = []
|
| 1862 | 2024-10-31T00:15:10.848Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 1863 | 2024-10-31T00:15:10.848Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 1864 | 2024-10-31T00:15:10.848Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:750
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 1865 | 2024-10-31T00:15:10.848Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 150
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:40636
req_id = 58e3ff63-0302-4f2b-98e8-35e85f3e0764
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 1866 | 2024-10-31T00:15:15.699Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:42260
|
| 1867 | 2024-10-31T00:15:15.700Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 535
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:42260
req_id = 4bc204aa-70cf-4aa2-8088-332f4b7f08dc
response_code = 200
uri = /vpc-routes
|
| 1868 | 2024-10-31T00:15:15.722Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 1869 | 2024-10-31T00:15:15.722Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 1153
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:42260
req_id = 84df363d-5565-46b0-afcc-1a6f742edaae
response_code = 204
uri = /eip-gateways
|
| 1870 | 2024-10-31T00:15:15.813Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 3108
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:42260
req_id = 50cefdd5-7162-476a-b97d-f763d9f99f8d
response_code = 204
uri = /vpc-routes
|
| 1871 | 2024-10-31T00:15:16.281Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:33315
|
| 1872 | 2024-10-31T00:15:16.282Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 47
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:33315
req_id = fb5a81c5-fac3-4e0e-8b45-332d4661a20e
response_code = 200
uri = /vpc-routes
|
| 1873 | 2024-10-31T00:15:16.397Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:34986
|
| 1874 | 2024-10-31T00:15:16.397Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 184
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:34986
req_id = fef3ae48-d56c-4521-9699-e550b91f132e
response_code = 200
uri = /v2p
|
| 1875 | 2024-10-31T00:15:16.447Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 1876 | 2024-10-31T00:15:16.447Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 67
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:33315
req_id = b3622922-c60a-4698-9f26-b4cb988cc064
response_code = 204
uri = /eip-gateways
|
| 1877 | 2024-10-31T00:15:16.505Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 172
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:33315
req_id = b3a07f14-25f5-4c5e-ae39-965dff06168b
response_code = 204
uri = /vpc-routes
|
| 1878 | 2024-10-31T00:15:16.627Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:34371
|
| 1879 | 2024-10-31T00:15:16.628Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 1880 | 2024-10-31T00:15:16.698Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 1881 | 2024-10-31T00:15:16.767Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 140002
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:34371
req_id = c8207f66-4015-4a65-8d2b-4baefbdafa29
response_code = 204
uri = /switch-ports
|
| 1882 | 2024-10-31T00:15:16.769Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 1470
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:34371
req_id = 725d98e0-d9e3-4a68-a0fd-0b1249f44136
response_code = 200
uri = /network-bootstore-config
|
| 1883 | 2024-10-31T00:15:16.804Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 2, current_generation: 1
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
| 1884 | 2024-10-31T00:15:16.804Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 1885 | 2024-10-31T00:15:16.805Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 1886 | 2024-10-31T00:15:16.807Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 2
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-sock-i86pc
|
| 1887 | 2024-10-31T00:15:16.807Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 3829
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:34371
req_id = 5d276de4-b1e2-4917-b5f6-ce71842f99b5
response_code = 204
uri = /network-bootstore-config
|
| 1888 | 2024-10-31T00:15:17.063Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:41619
|
| 1889 | 2024-10-31T00:15:17.063Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 60
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:41619
req_id = 6a461945-b2fc-43fc-9359-3f41f6490737
response_code = 200
uri = /vpc-routes
|
| 1890 | 2024-10-31T00:15:17.096Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 1891 | 2024-10-31T00:15:17.096Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 97
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:41619
req_id = 61a02718-a8bb-41f5-864a-4a7ee44dd55f
response_code = 204
uri = /eip-gateways
|
| 1892 | 2024-10-31T00:15:17.134Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:59344
|
| 1893 | 2024-10-31T00:15:17.134Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 62
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:59344
req_id = 604a4501-7d70-40be-b44c-e5ca51949eb0
response_code = 200
uri = /vpc-routes
|
| 1894 | 2024-10-31T00:15:17.219Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 1895 | 2024-10-31T00:15:17.219Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 85
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:59344
req_id = a43dae92-4620-4720-97a8-30756f0e7b03
response_code = 204
uri = /eip-gateways
|
| 1896 | 2024-10-31T00:15:17.352Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 174
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:41619
req_id = 4c98b93a-1bec-4660-9ec1-7334884258b0
response_code = 204
uri = /vpc-routes
|
| 1897 | 2024-10-31T00:15:17.394Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 173
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:59344
req_id = 7350159b-758f-478a-bc6e-05443bd79d44
response_code = 204
uri = /vpc-routes
|
| 1898 | 2024-10-31T00:15:17.424Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:42294
|
| 1899 | 2024-10-31T00:15:17.424Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 130
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:42294
req_id = 3f66707b-d5b2-4766-a070-afcaa52de888
response_code = 200
uri = /v2p
|
| 1900 | 2024-10-31T00:15:17.429Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:36286
|
| 1901 | 2024-10-31T00:15:17.429Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 1902 | 2024-10-31T00:15:17.474Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 1903 | 2024-10-31T00:15:17.541Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 111847
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:36286
req_id = 7d3b056b-e450-407e-a639-bcba9d3d8239
response_code = 204
uri = /switch-ports
|
| 1904 | 2024-10-31T00:15:17.541Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:60352
|
| 1905 | 2024-10-31T00:15:17.541Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:49324
|
| 1906 | 2024-10-31T00:15:17.542Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 239
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:36286
req_id = a0d50dd2-ae59-44eb-811c-f28745e1fdfd
response_code = 200
uri = /network-bootstore-config
|
| 1907 | 2024-10-31T00:15:17.542Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 1908 | 2024-10-31T00:15:17.547Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 5383
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:49324
req_id = bf590cdd-8acb-423c-bc2c-387f99713ea8
response_code = 200
uri = /v2p
|
| 1909 | 2024-10-31T00:15:17.587Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 1910 | 2024-10-31T00:15:17.653Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 111320
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:60352
req_id = aefa3b7f-688e-4adb-9a70-68db3b815dc0
response_code = 204
uri = /switch-ports
|
| 1911 | 2024-10-31T00:15:17.653Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 148
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:60352
req_id = 26748ca7-f6a7-4272-8edc-fc6d4e6ddc0e
response_code = 200
uri = /network-bootstore-config
|
| 1912 | 2024-10-31T00:15:30.708Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:47127
|
| 1913 | 2024-10-31T00:15:30.708Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 35
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:47127
req_id = 4403c0d4-a303-4436-a0d6-c2e033a88ae9
response_code = 200
uri = /baseboard
|
| 1914 | 2024-10-31T00:15:46.229Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:56966
|
| 1915 | 2024-10-31T00:15:46.229Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 86
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:56966
req_id = ccc8c684-e9e4-4525-938b-620ddb69a0b5
response_code = 200
uri = /vpc-routes
|
| 1916 | 2024-10-31T00:15:46.291Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 1917 | 2024-10-31T00:15:46.291Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 106
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:56966
req_id = 4a941008-3932-476e-bd27-074280b764d6
response_code = 204
uri = /eip-gateways
|
| 1918 | 2024-10-31T00:15:46.301Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 208
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:56966
req_id = 9d63f4bc-b15e-466a-ac74-4c06b4d8adbf
response_code = 204
uri = /vpc-routes
|
| 1919 | 2024-10-31T00:15:46.408Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:64119
|
| 1920 | 2024-10-31T00:15:46.408Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 108
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:64119
req_id = 482abf45-2255-48fd-a35b-ee820fa3c126
response_code = 200
uri = /v2p
|
| 1921 | 2024-10-31T00:15:46.423Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:60713
|
| 1922 | 2024-10-31T00:15:46.423Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 1923 | 2024-10-31T00:15:46.464Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 1924 | 2024-10-31T00:15:46.521Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 97996
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:60713
req_id = 574c7b10-12b4-47e2-a94b-478f73d8c0ec
response_code = 204
uri = /switch-ports
|
| 1925 | 2024-10-31T00:15:46.521Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 119
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:60713
req_id = a806bec3-2160-499b-a324-7dd116b991a4
response_code = 200
uri = /network-bootstore-config
|
| 1926 | 2024-10-31T00:15:47.088Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:59381
|
| 1927 | 2024-10-31T00:15:47.088Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 83
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:59381
req_id = 5be28f52-32f7-42c5-85fc-eaf32ee25806
response_code = 200
uri = /vpc-routes
|
| 1928 | 2024-10-31T00:15:47.143Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 1929 | 2024-10-31T00:15:47.143Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 86
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:59381
req_id = cb1dd103-a044-4bf7-959b-0cdb1e8ca276
response_code = 204
uri = /eip-gateways
|
| 1930 | 2024-10-31T00:15:47.155Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 271
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:59381
req_id = f05a1d75-ab22-4102-b9fe-f0278149fa20
response_code = 204
uri = /vpc-routes
|
| 1931 | 2024-10-31T00:15:47.255Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:41003
|
| 1932 | 2024-10-31T00:15:47.255Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 139
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:41003
req_id = 26c2e6cc-9a30-40f3-92fd-bb7d73f98829
response_code = 200
uri = /v2p
|
| 1933 | 2024-10-31T00:15:47.279Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:37141
|
| 1934 | 2024-10-31T00:15:47.279Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 1935 | 2024-10-31T00:15:47.316Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 1936 | 2024-10-31T00:15:47.392Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 113519
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:37141
req_id = 281eb10e-79fd-429f-a541-9cd59fd04ee6
response_code = 204
uri = /switch-ports
|
| 1937 | 2024-10-31T00:15:47.392Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:50752
|
| 1938 | 2024-10-31T00:15:47.398Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 5474
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:37141
req_id = 382ac351-9b43-43f5-801a-fe5eec4b0f76
response_code = 200
uri = /network-bootstore-config
|
| 1939 | 2024-10-31T00:15:47.398Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 571
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:50752
req_id = 41aaaa1e-60d4-45c1-bd4c-1ae878394403
response_code = 200
uri = /vpc-routes
|
| 1940 | 2024-10-31T00:15:47.447Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 1941 | 2024-10-31T00:15:47.447Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 97
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:50752
req_id = adcc8c90-bc37-413d-a7d7-396fc3e461d7
response_code = 204
uri = /eip-gateways
|
| 1942 | 2024-10-31T00:15:47.457Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 264
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:50752
req_id = 3a77c446-b55a-4da2-96c7-74c4f9b5630c
response_code = 204
uri = /vpc-routes
|
| 1943 | 2024-10-31T00:15:47.472Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:59884
|
| 1944 | 2024-10-31T00:15:47.472Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 1945 | 2024-10-31T00:15:47.508Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 1946 | 2024-10-31T00:15:47.565Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 92938
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:59884
req_id = 9e16bb8a-c2a0-44d5-a765-ad0081004eef
response_code = 204
uri = /switch-ports
|
| 1947 | 2024-10-31T00:15:47.565Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:44936
|
| 1948 | 2024-10-31T00:15:47.565Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 166
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:44936
req_id = 0cea5cae-d213-411a-8da1-8747ddb3c2fb
response_code = 200
uri = /v2p
|
| 1949 | 2024-10-31T00:15:47.565Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 158
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:59884
req_id = dc0c0e9a-5c11-477e-ada3-bfc777699b4b
response_code = 200
uri = /network-bootstore-config
|
| 1950 | 2024-10-31T00:16:00.741Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56363
|
| 1951 | 2024-10-31T00:16:00.741Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 58
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56363
req_id = 0ff2c40f-47db-4916-9d95-464805a56418
response_code = 200
uri = /baseboard
|
| 1952 | 2024-10-31T00:16:16.159Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:53173
|
| 1953 | 2024-10-31T00:16:16.159Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 78
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:53173
req_id = c4c885d2-3328-467e-a5f4-dec5f9574e1c
response_code = 200
uri = /vpc-routes
|
| 1954 | 2024-10-31T00:16:16.204Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 1955 | 2024-10-31T00:16:16.205Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 93
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:53173
req_id = 8e920e5f-c0d5-4c7e-b15a-45ce676ef62a
response_code = 204
uri = /eip-gateways
|
| 1956 | 2024-10-31T00:16:16.213Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 178
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:53173
req_id = 4854324e-7365-4825-81b6-e78044319fd7
response_code = 204
uri = /vpc-routes
|
| 1957 | 2024-10-31T00:16:16.369Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:54059
|
| 1958 | 2024-10-31T00:16:16.369Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 123
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:54059
req_id = 51c5205d-b5ed-468b-8da2-8406515522e4
response_code = 200
uri = /v2p
|
| 1959 | 2024-10-31T00:16:16.416Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:37056
|
| 1960 | 2024-10-31T00:16:16.416Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 1961 | 2024-10-31T00:16:16.453Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 1962 | 2024-10-31T00:16:16.513Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 96839
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:37056
req_id = ba72233e-3edd-4549-8317-b01d185b538a
response_code = 204
uri = /switch-ports
|
| 1963 | 2024-10-31T00:16:16.514Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 117
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:37056
req_id = f006bf94-5ed0-4d6a-b956-3d868e659180
response_code = 200
uri = /network-bootstore-config
|
| 1964 | 2024-10-31T00:16:16.924Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:55802
|
| 1965 | 2024-10-31T00:16:16.924Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 86
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:55802
req_id = e1b27756-7745-4b30-8d89-bfe776da047d
response_code = 200
uri = /vpc-routes
|
| 1966 | 2024-10-31T00:16:16.978Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 1967 | 2024-10-31T00:16:16.978Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 107
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:55802
req_id = 91b796c6-f31d-456b-baf8-1d514d348862
response_code = 204
uri = /eip-gateways
|
| 1968 | 2024-10-31T00:16:16.989Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 244
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:55802
req_id = 40f98c0f-3605-4ad2-9b40-bcf7dd6f2494
response_code = 204
uri = /vpc-routes
|
| 1969 | 2024-10-31T00:16:17.204Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:46862
|
| 1970 | 2024-10-31T00:16:17.204Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 203
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:46862
req_id = 2c27c882-8d8c-42f6-8603-417ab04b864b
response_code = 200
uri = /v2p
|
| 1971 | 2024-10-31T00:16:17.212Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:62701
|
| 1972 | 2024-10-31T00:16:17.212Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 51
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:62701
req_id = 8afda944-e752-4f14-8b73-66f35c54829c
response_code = 200
uri = /vpc-routes
|
| 1973 | 2024-10-31T00:16:17.227Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:62177
|
| 1974 | 2024-10-31T00:16:17.227Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 1975 | 2024-10-31T00:16:17.269Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 1976 | 2024-10-31T00:16:17.336Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 109283
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:62177
req_id = 1767e4c3-098e-423d-8819-0a4bcb2b0c04
response_code = 204
uri = /switch-ports
|
| 1977 | 2024-10-31T00:16:17.336Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:49193
|
| 1978 | 2024-10-31T00:16:17.336Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 1979 | 2024-10-31T00:16:17.336Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 142
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:62701
req_id = 65ea1c6e-084b-4658-9a52-78e1ff22038f
response_code = 204
uri = /eip-gateways
|
| 1980 | 2024-10-31T00:16:17.337Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 278
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:62177
req_id = 4a0d758f-365f-4d48-b39f-4f5c65ccfaaa
response_code = 200
uri = /network-bootstore-config
|
| 1981 | 2024-10-31T00:16:17.337Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 1982 | 2024-10-31T00:16:17.347Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 351
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:62701
req_id = 38902c43-3044-4c06-8e60-3b5864bb0292
response_code = 204
uri = /vpc-routes
|
| 1983 | 2024-10-31T00:16:17.377Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 1984 | 2024-10-31T00:16:17.409Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:55592
|
| 1985 | 2024-10-31T00:16:17.410Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 155
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:55592
req_id = 43d52f2c-6efd-4437-81f6-cfaabf19a9da
response_code = 200
uri = /v2p
|
| 1986 | 2024-10-31T00:16:17.437Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 99951
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:49193
req_id = 56bf7302-d008-4263-9002-782f9713b11c
response_code = 204
uri = /switch-ports
|
| 1987 | 2024-10-31T00:16:17.437Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 143
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:49193
req_id = 485b52c5-3a53-492a-8d10-9e3ba6b9aad1
response_code = 200
uri = /network-bootstore-config
|
| 1988 | 2024-10-31T00:16:30.778Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:41939
|
| 1989 | 2024-10-31T00:16:30.778Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 37
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:41939
req_id = 5fecfe4f-ff97-4dc3-b99f-883bd4057693
response_code = 200
uri = /baseboard
|
| 1990 | 2024-10-31T00:16:46.358Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:56607
|
| 1991 | 2024-10-31T00:16:46.358Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 71
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:56607
req_id = ff869294-e24d-4da3-983b-adf272890e9d
response_code = 200
uri = /vpc-routes
|
| 1992 | 2024-10-31T00:16:46.405Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 1993 | 2024-10-31T00:16:46.405Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 74
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:56607
req_id = 668bb66d-4d53-43af-96b2-127e62cf17c7
response_code = 204
uri = /eip-gateways
|
| 1994 | 2024-10-31T00:16:46.418Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:35041
|
| 1995 | 2024-10-31T00:16:46.418Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 129
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:35041
req_id = c35af1dd-2db7-45f5-85fa-08bef3a76c5e
response_code = 200
uri = /v2p
|
| 1996 | 2024-10-31T00:16:46.424Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 133
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:56607
req_id = c6ccde08-eecf-45e2-88a6-75dc40679f87
response_code = 204
uri = /vpc-routes
|
| 1997 | 2024-10-31T00:16:46.500Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:56109
|
| 1998 | 2024-10-31T00:16:46.500Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 1999 | 2024-10-31T00:16:46.540Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2000 | 2024-10-31T00:16:46.598Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 98418
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:56109
req_id = 04ca7f76-1341-40f5-9b20-3fb115a5a5e5
response_code = 204
uri = /switch-ports
|
| 2001 | 2024-10-31T00:16:46.599Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 111
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:56109
req_id = 08c2a65c-b410-46ac-bf73-fbcb2001fbd8
response_code = 200
uri = /network-bootstore-config
|
| 2002 | 2024-10-31T00:16:47.052Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:44093
|
| 2003 | 2024-10-31T00:16:47.052Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 117
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:44093
req_id = ca75707f-ead7-4ce5-97aa-71369ddbe92b
response_code = 200
uri = /vpc-routes
|
| 2004 | 2024-10-31T00:16:47.155Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 2005 | 2024-10-31T00:16:47.155Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 113
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:44093
req_id = 063c39e5-29b5-4c8f-900f-0006471b346e
response_code = 204
uri = /eip-gateways
|
| 2006 | 2024-10-31T00:16:47.169Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 222
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:44093
req_id = 64204f58-fb39-4d5f-b732-d6f8865b98f2
response_code = 204
uri = /vpc-routes
|
| 2007 | 2024-10-31T00:16:47.317Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:52084
|
| 2008 | 2024-10-31T00:16:47.318Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 142
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:52084
req_id = e1d57f0f-8530-46aa-a55a-a27a51d8057d
response_code = 200
uri = /v2p
|
| 2009 | 2024-10-31T00:16:47.342Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:42679
|
| 2010 | 2024-10-31T00:16:47.342Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 2011 | 2024-10-31T00:16:47.380Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2012 | 2024-10-31T00:16:47.437Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 94992
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:42679
req_id = b3aeae6b-d9cb-42ad-b90a-34cd7ccd456a
response_code = 204
uri = /switch-ports
|
| 2013 | 2024-10-31T00:16:47.438Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:36371
|
| 2014 | 2024-10-31T00:16:47.438Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:55167
|
| 2015 | 2024-10-31T00:16:47.438Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 218
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:42679
req_id = 92ea80c9-3ebd-4b50-9c51-45b6b028258c
response_code = 200
uri = /network-bootstore-config
|
| 2016 | 2024-10-31T00:16:47.438Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 151
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:36371
req_id = 39dac58a-703f-4288-afaa-587a5cd779da
response_code = 200
uri = /vpc-routes
|
| 2017 | 2024-10-31T00:16:47.438Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 238
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:55167
req_id = b5636f17-f996-4dd0-b4fe-a505f79c496d
response_code = 200
uri = /v2p
|
| 2018 | 2024-10-31T00:16:47.488Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 2019 | 2024-10-31T00:16:47.488Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 101
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:36371
req_id = b34f8753-2f91-4039-a8bf-ad1d73c79b5d
response_code = 204
uri = /eip-gateways
|
| 2020 | 2024-10-31T00:16:47.498Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 267
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:36371
req_id = 4d8ccea1-c778-4f58-888b-a279c75a916c
response_code = 204
uri = /vpc-routes
|
| 2021 | 2024-10-31T00:16:47.623Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:54259
|
| 2022 | 2024-10-31T00:16:47.623Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 2023 | 2024-10-31T00:16:47.659Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2024 | 2024-10-31T00:16:47.710Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 87201
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:54259
req_id = e173e7f7-d033-4e80-a813-bbfc77517c8f
response_code = 204
uri = /switch-ports
|
| 2025 | 2024-10-31T00:16:47.711Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 207
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:54259
req_id = ddc2bb86-806b-4000-84b4-c3368720124a
response_code = 200
uri = /network-bootstore-config
|
| 2026 | 2024-10-31T00:17:00.810Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45644
|
| 2027 | 2024-10-31T00:17:00.810Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 32
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45644
req_id = 89e1b494-0426-4470-97f9-5d7e3a2bf21b
response_code = 200
uri = /baseboard
|
| 2028 | 2024-10-31T00:17:16.209Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:36156
|
| 2029 | 2024-10-31T00:17:16.209Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 87
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:36156
req_id = 2b185e99-ce5f-4e9a-8723-043116643412
response_code = 200
uri = /vpc-routes
|
| 2030 | 2024-10-31T00:17:16.251Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 2031 | 2024-10-31T00:17:16.251Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 103
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:36156
req_id = 2e453a5d-13cb-4f6f-ada8-b2e36befa97f
response_code = 204
uri = /eip-gateways
|
| 2032 | 2024-10-31T00:17:16.264Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 203
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:36156
req_id = f943b0c0-a087-475a-a04e-ea10313bbe03
response_code = 204
uri = /vpc-routes
|
| 2033 | 2024-10-31T00:17:16.305Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:47558
|
| 2034 | 2024-10-31T00:17:16.305Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 149
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:47558
req_id = 0962c029-2293-4d6b-99db-c49eb9c9393c
response_code = 200
uri = /v2p
|
| 2035 | 2024-10-31T00:17:16.359Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:62081
|
| 2036 | 2024-10-31T00:17:16.359Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 2037 | 2024-10-31T00:17:16.397Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2038 | 2024-10-31T00:17:16.449Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 89899
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:62081
req_id = 915ec286-c4b5-43e3-8c12-f095a8db0858
response_code = 204
uri = /switch-ports
|
| 2039 | 2024-10-31T00:17:16.449Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 109
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:62081
req_id = e7d693ca-f5e0-481c-b271-accf0a74304b
response_code = 200
uri = /network-bootstore-config
|
| 2040 | 2024-10-31T00:17:17.048Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:48385
|
| 2041 | 2024-10-31T00:17:17.048Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 89
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:48385
req_id = 36be1ee2-0bab-47b8-bb46-e08a803cc9d9
response_code = 200
uri = /vpc-routes
|
| 2042 | 2024-10-31T00:17:17.092Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 2043 | 2024-10-31T00:17:17.092Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 104
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:48385
req_id = 26067b8f-d7da-44a5-b71f-e5573e8bf504
response_code = 204
uri = /eip-gateways
|
| 2044 | 2024-10-31T00:17:17.103Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 323
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:48385
req_id = c2fda985-f297-4a96-b4eb-8315a2125083
response_code = 204
uri = /vpc-routes
|
| 2045 | 2024-10-31T00:17:17.154Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:65198
|
| 2046 | 2024-10-31T00:17:17.154Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 73
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:65198
req_id = 27188a28-9d78-4a55-91f6-e93bdd1403e8
response_code = 200
uri = /vpc-routes
|
| 2047 | 2024-10-31T00:17:17.164Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:51357
|
| 2048 | 2024-10-31T00:17:17.164Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 136
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:51357
req_id = 4a14f129-5596-48b7-a320-60f9601f3ad6
response_code = 200
uri = /v2p
|
| 2049 | 2024-10-31T00:17:17.173Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 2050 | 2024-10-31T00:17:17.173Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 57
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:65198
req_id = 6537f32e-22db-4d71-87d0-42c43c945047
response_code = 204
uri = /eip-gateways
|
| 2051 | 2024-10-31T00:17:17.181Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 189
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:65198
req_id = 19789701-1a86-4d56-aa26-01ec1e2eddfc
response_code = 204
uri = /vpc-routes
|
| 2052 | 2024-10-31T00:17:17.187Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:62444
|
| 2053 | 2024-10-31T00:17:17.187Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 2054 | 2024-10-31T00:17:17.230Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2055 | 2024-10-31T00:17:17.293Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 106234
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:62444
req_id = bffded0e-0392-49c0-befc-544f2f4adec2
response_code = 204
uri = /switch-ports
|
| 2056 | 2024-10-31T00:17:17.293Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 104
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:62444
req_id = 40b7898d-5d8c-4c30-a8e0-724d2896aefb
response_code = 200
uri = /network-bootstore-config
|
| 2057 | 2024-10-31T00:17:17.308Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:51415
|
| 2058 | 2024-10-31T00:17:17.308Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 2059 | 2024-10-31T00:17:17.346Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2060 | 2024-10-31T00:17:17.404Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 96280
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:51415
req_id = 5465e6b0-b3de-4986-90d1-d37b7e38dd6f
response_code = 204
uri = /switch-ports
|
| 2061 | 2024-10-31T00:17:17.405Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:55034
|
| 2062 | 2024-10-31T00:17:17.405Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 234
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:51415
req_id = 6c7994c7-ea83-4437-bcb7-c6232a7d0922
response_code = 200
uri = /network-bootstore-config
|
| 2063 | 2024-10-31T00:17:17.405Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 190
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:55034
req_id = 3251646a-6a25-4f5c-8a0c-83eea247690c
response_code = 200
uri = /v2p
|
| 2064 | 2024-10-31T00:17:30.843Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:61223
|
| 2065 | 2024-10-31T00:17:30.843Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 32
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:61223
req_id = e3108823-5ff3-4561-8ca5-73c81e9f3d2f
response_code = 200
uri = /baseboard
|
| 2066 | 2024-10-31T00:17:39.719Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 2067 | 2024-10-31T00:17:46.395Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:47207
|
| 2068 | 2024-10-31T00:17:46.396Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 71
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:47207
req_id = 9c6e3d99-b836-433d-a11b-d95c8bb25ebf
response_code = 200
uri = /vpc-routes
|
| 2069 | 2024-10-31T00:17:46.445Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 2070 | 2024-10-31T00:17:46.445Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 90
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:47207
req_id = d9bb2dcb-345c-4fb1-a9b0-3414f33a1e9e
response_code = 204
uri = /eip-gateways
|
| 2071 | 2024-10-31T00:17:46.455Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 197
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:47207
req_id = a34775b8-ad76-41d5-8e9d-74c3ef498574
response_code = 204
uri = /vpc-routes
|
| 2072 | 2024-10-31T00:17:46.535Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:54465
|
| 2073 | 2024-10-31T00:17:46.535Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 150
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:54465
req_id = bac86bd6-e848-4b08-bf19-975b35051db2
response_code = 200
uri = /v2p
|
| 2074 | 2024-10-31T00:17:46.567Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:52986
|
| 2075 | 2024-10-31T00:17:46.567Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 2076 | 2024-10-31T00:17:46.601Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2077 | 2024-10-31T00:17:46.656Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 89451
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:52986
req_id = bf7f5237-9618-42dc-bb58-17b13fd463c7
response_code = 204
uri = /switch-ports
|
| 2078 | 2024-10-31T00:17:46.657Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 141
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:52986
req_id = 859282ba-2484-464c-b0c1-f9574b982c93
response_code = 200
uri = /network-bootstore-config
|
| 2079 | 2024-10-31T00:17:47.163Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:58215
|
| 2080 | 2024-10-31T00:17:47.164Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 93
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:58215
req_id = 2907c529-b93c-4629-b2bc-d7fea9d0a620
response_code = 200
uri = /vpc-routes
|
| 2081 | 2024-10-31T00:17:47.220Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 2082 | 2024-10-31T00:17:47.220Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 95
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:58215
req_id = 181c9357-f851-4bc8-92eb-af6d3274e1e8
response_code = 204
uri = /eip-gateways
|
| 2083 | 2024-10-31T00:17:47.239Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:58300
|
| 2084 | 2024-10-31T00:17:47.239Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 164
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:58300
req_id = 920fcf06-ab59-4a11-8730-d555c63e8c02
response_code = 200
uri = /v2p
|
| 2085 | 2024-10-31T00:17:47.249Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 176
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:58215
req_id = ed1efd67-cca8-479f-bc54-601dc9c7a3ea
response_code = 204
uri = /vpc-routes
|
| 2086 | 2024-10-31T00:17:47.298Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:42144
|
| 2087 | 2024-10-31T00:17:47.298Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 2088 | 2024-10-31T00:17:47.335Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2089 | 2024-10-31T00:17:47.390Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 92147
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:42144
req_id = bc7de81c-7896-4ab7-b970-bcdf3a1447cd
response_code = 204
uri = /switch-ports
|
| 2090 | 2024-10-31T00:17:47.390Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 118
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:42144
req_id = 931a01bc-75f5-44b6-82c4-193e479e1de4
response_code = 200
uri = /network-bootstore-config
|
| 2091 | 2024-10-31T00:17:47.497Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:60714
|
| 2092 | 2024-10-31T00:17:47.497Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 30
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:60714
req_id = b5f539a6-55fc-47c4-890c-da17af788782
response_code = 200
uri = /vpc-routes
|
| 2093 | 2024-10-31T00:17:47.517Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 2094 | 2024-10-31T00:17:47.517Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 57
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:60714
req_id = f0d4c23a-765f-45cf-8b11-02e9ccb819fe
response_code = 204
uri = /eip-gateways
|
| 2095 | 2024-10-31T00:17:47.526Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 167
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:60714
req_id = 29bdf832-f4f7-4935-8bcf-4d2c5ab7451e
response_code = 204
uri = /vpc-routes
|
| 2096 | 2024-10-31T00:17:47.613Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:56389
|
| 2097 | 2024-10-31T00:17:47.613Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 135
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:56389
req_id = c7ee502a-7c61-4a5b-9f4d-8f49126a3ae9
response_code = 200
uri = /v2p
|
| 2098 | 2024-10-31T00:17:47.615Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:65297
|
| 2099 | 2024-10-31T00:17:47.615Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 2100 | 2024-10-31T00:17:47.658Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2101 | 2024-10-31T00:17:47.718Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 103012
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:65297
req_id = 59db88dd-b082-493e-8e63-002ec500a158
response_code = 204
uri = /switch-ports
|
| 2102 | 2024-10-31T00:17:47.719Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 114
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:65297
req_id = 4111934a-879c-4a64-9de9-98a326f5abae
response_code = 200
uri = /network-bootstore-config
|
| 2103 | 2024-10-31T00:18:00.878Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:46578
|
| 2104 | 2024-10-31T00:18:00.878Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 35
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:46578
req_id = 0dc9be49-b60d-44ad-8dd9-add422e8e9e2
response_code = 200
uri = /baseboard
|
| 2105 | 2024-10-31T00:18:16.183Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:63559
|
| 2106 | 2024-10-31T00:18:16.184Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 78
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:63559
req_id = 2f185e7c-9b12-462c-b747-1845f28faf07
response_code = 200
uri = /vpc-routes
|
| 2107 | 2024-10-31T00:18:16.225Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 2108 | 2024-10-31T00:18:16.225Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 88
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:63559
req_id = d1871b01-8084-4ac4-8b41-5f70a0299ef7
response_code = 204
uri = /eip-gateways
|
| 2109 | 2024-10-31T00:18:16.233Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 175
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:63559
req_id = 55ba7333-16f5-4bb8-8ef1-ce34ec9b3397
response_code = 204
uri = /vpc-routes
|
| 2110 | 2024-10-31T00:18:16.293Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:45621
|
| 2111 | 2024-10-31T00:18:16.293Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 2112 | 2024-10-31T00:18:16.332Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2113 | 2024-10-31T00:18:16.386Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 92749
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:45621
req_id = 7acb0d66-58a3-4999-bac4-e9d47cb0a26a
response_code = 204
uri = /switch-ports
|
| 2114 | 2024-10-31T00:18:16.386Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:64651
|
| 2115 | 2024-10-31T00:18:16.386Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 120
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:45621
req_id = 9e7b278e-e23e-47d5-a008-fe371905d6e5
response_code = 200
uri = /network-bootstore-config
|
| 2116 | 2024-10-31T00:18:16.387Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 145
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:64651
req_id = 27faff21-9101-4880-9e51-69ffacd884d9
response_code = 200
uri = /v2p
|
| 2117 | 2024-10-31T00:18:17.041Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:63150
|
| 2118 | 2024-10-31T00:18:17.041Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 94
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:63150
req_id = 0f46f3e4-64cb-480e-a744-2afa0cdd4f1b
response_code = 200
uri = /vpc-routes
|
| 2119 | 2024-10-31T00:18:17.064Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:63825
|
| 2120 | 2024-10-31T00:18:17.064Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 183
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:63825
req_id = aca78501-7e48-44f5-9a6b-b99ce8bc2183
response_code = 200
uri = /v2p
|
| 2121 | 2024-10-31T00:18:17.092Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 2122 | 2024-10-31T00:18:17.092Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 80
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:63150
req_id = a8a2bce8-ed24-45d1-9782-fcf7d958019b
response_code = 204
uri = /eip-gateways
|
| 2123 | 2024-10-31T00:18:17.101Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 200
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:63150
req_id = b3bb0dd2-38cd-46eb-ab77-1c9fda3f0af4
response_code = 204
uri = /vpc-routes
|
| 2124 | 2024-10-31T00:18:17.156Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:59674
|
| 2125 | 2024-10-31T00:18:17.156Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 2126 | 2024-10-31T00:18:17.196Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2127 | 2024-10-31T00:18:17.255Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 98729
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:59674
req_id = fc165c57-6eb2-42cd-abac-1886a922e2d1
response_code = 204
uri = /switch-ports
|
| 2128 | 2024-10-31T00:18:17.255Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 119
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:59674
req_id = 9b53f7f9-df3b-455b-a4d7-e40f3cc081a4
response_code = 200
uri = /network-bootstore-config
|
| 2129 | 2024-10-31T00:18:17.303Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:61251
|
| 2130 | 2024-10-31T00:18:17.304Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 67
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:61251
req_id = 57143d5d-9cff-4882-83a2-ece99d79ef30
response_code = 200
uri = /vpc-routes
|
| 2131 | 2024-10-31T00:18:17.329Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 2132 | 2024-10-31T00:18:17.329Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 75
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:61251
req_id = 1b3fc3d8-eab4-4a10-b179-feed38f800d8
response_code = 204
uri = /eip-gateways
|
| 2133 | 2024-10-31T00:18:17.338Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 211
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:61251
req_id = 75f6dd86-87fb-44a7-888b-c80136d2efaf
response_code = 204
uri = /vpc-routes
|
| 2134 | 2024-10-31T00:18:17.363Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:55033
|
| 2135 | 2024-10-31T00:18:17.363Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 2136 | 2024-10-31T00:18:17.403Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2137 | 2024-10-31T00:18:17.459Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 96134
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:55033
req_id = 15a847de-f8bd-4233-99d9-436fef4e9858
response_code = 204
uri = /switch-ports
|
| 2138 | 2024-10-31T00:18:17.459Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:64593
|
| 2139 | 2024-10-31T00:18:17.459Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 113
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:55033
req_id = 47c6cb76-7bee-4fe5-ae51-38295fa1395d
response_code = 200
uri = /network-bootstore-config
|
| 2140 | 2024-10-31T00:18:17.460Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 136
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:64593
req_id = a2ad7f8d-77b8-4a9b-87c8-6098c5c3832d
response_code = 200
uri = /v2p
|
| 2141 | 2024-10-31T00:18:30.911Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:60844
|
| 2142 | 2024-10-31T00:18:30.911Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 42
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:60844
req_id = 4035a558-59a9-4c28-8b0d-63995b96e56a
response_code = 200
uri = /baseboard
|
| 2143 | 2024-10-31T00:18:38.384Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:55035
|
| 2144 | 2024-10-31T00:18:38.384Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 160
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:55035
req_id = 04406ea5-b447-4ce1-8f17-10682b00a472
response_code = 200
uri = /v2p
|
| 2145 | 2024-10-31T00:18:38.386Z | INFO | SledAgent (PortManager): Mapping virtual NIC to physical host
file = illumos-utils/src/opte/port_manager.rs:838
mapping = VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(15018464) }
|
| 2146 | 2024-10-31T00:18:38.386Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 101
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:55035
req_id = beb61adc-ea44-4bfc-a3c9-0db6125db158
response_code = 204
uri = /v2p
|
| 2147 | 2024-10-31T00:18:38.565Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:40776
|
| 2148 | 2024-10-31T00:18:38.572Z | INFO | SledAgent (InstanceManager): ensuring instance is registered
file = sled-agent/src/instance_manager.rs:555
hardware = InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname("localshark") }, nics: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(15018464))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: "nvme", name: "disk-f15630e35db", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: cb360434-786a-4a5f-b2cb-17b2f9af1a5f, read_only_parent: Some(Volume { block_size: 512, id: f4843286-27a6-4342-aff1-c0d8c16c1dd7, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 246c3072-1690-4f6f-adf8-5c44a3742362, key: Some("7xNaLJhQXfLIWc34eKqxqf5Z6MSLemeqY4kg6Q8IOl8="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::13]:19001", "[fd00:1122:3344:101::16]:19001", "[fd00:1122:3344:101::12]:19002"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: cb360434-786a-4a5f-b2cb-17b2f9af1a5f, key: Some("MFxAp2a/h6Pto4lsEvm8yEWCoSW44SJldq3tI6+4v0k="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::1a]:19000", "[fd00:1122:3344:101::15]:19000", "[fd00:1122:3344:101::18]:19000"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: "disk-f15630e35db" }] }), cloud_init_bytes: Some(..) }
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
metadata = InstanceMetadata { silo_id: 581b9a38-aa7e-48a2-bd72-3e109217b461, project_id: b8fce7e8-d453-462f-adc8-3733e1c4452d }
migration_id = None
propolis_addr = [fd00:1122:3344:101::1:0]:12400
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
vmm_runtime = VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-10-31T00:18:38.565403590Z }
|
| 2149 | 2024-10-31T00:18:38.572Z | INFO | SledAgent (InstanceManager): registering new instance
file = sled-agent/src/instance_manager.rs:588
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
migration_id = None
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2150 | 2024-10-31T00:18:38.572Z | INFO | SledAgent (InstanceManager): initializing new Instance
file = sled-agent/src/instance.rs:1281
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
migration_id = None
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
state = InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname("localshark") }, nics: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(15018464))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name("net0"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: "nvme", name: "disk-f15630e35db", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: cb360434-786a-4a5f-b2cb-17b2f9af1a5f, read_only_parent: Some(Volume { block_size: 512, id: f4843286-27a6-4342-aff1-c0d8c16c1dd7, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 246c3072-1690-4f6f-adf8-5c44a3742362, key: Some("7xNaLJhQXfLIWc34eKqxqf5Z6MSLemeqY4kg6Q8IOl8="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::13]:19001", "[fd00:1122:3344:101::16]:19001", "[fd00:1122:3344:101::12]:19002"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: cb360434-786a-4a5f-b2cb-17b2f9af1a5f, key: Some("MFxAp2a/h6Pto4lsEvm8yEWCoSW44SJldq3tI6+4v0k="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::1a]:19000", "[fd00:1122:3344:101::15]:19000", "[fd00:1122:3344:101::18]:19000"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: "disk-f15630e35db" }] }), cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-10-31T00:18:38.565403590Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }
|
| 2151 | 2024-10-31T00:18:38.574Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 8700
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:40776
req_id = ebd08d81-2c4d-4fb8-91dc-0f8291ef6f01
response_code = 200
uri = /vmms/8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2152 | 2024-10-31T00:18:38.686Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:43117
|
| 2153 | 2024-10-31T00:18:38.687Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 147
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:43117
req_id = 3031d5d6-ac8e-4ec0-9d5f-ee99090e4996
response_code = 200
uri = /vmms/8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd/state
|
| 2154 | 2024-10-31T00:18:38.691Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:430
port = Port { inner: PortInner(PortData { name: "opte6", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 0, 0]), slot: 0, vni: Vni { inner: 15018464 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }
|
| 2155 | 2024-10-31T00:18:38.691Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 2156 | 2024-10-31T00:18:38.729Z | INFO | SledAgent (InstanceManager): Configuring new Omicron zone: oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
file = illumos-utils/src/zone.rs:325
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2157 | 2024-10-31T00:18:38.751Z | INFO | SledAgent (InstanceManager): Installing Omicron zone: oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
file = illumos-utils/src/zone.rs:363
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2158 | 2024-10-31T00:18:38.763Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:57528
|
| 2159 | 2024-10-31T00:18:38.763Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 102
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:57528
req_id = 1ea72172-3257-4295-b006-f3d2f79f2a00
response_code = 200
uri = /vpc-routes
|
| 2160 | 2024-10-31T00:18:38.789Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {487b7899-43a4-486a-bd8d-ca0047fd7478: {10.151.2.175: {1865c5a2-f061-4256-aaff-e92277867544}, 10.151.2.176: {1865c5a2-f061-4256-aaff-e92277867544}}} }
|
| 2161 | 2024-10-31T00:18:40.233Z | INFO | SledAgent (InstanceManager): Profile for oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="system/illumos/propolis-server">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlInstance0'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="listen_addr" value='fd00:1122:3344:101::1:0'/>
<propval type="astring" name="listen_port" value='12400'/>
<propval type="astring" name="metric_addr" value='dns'/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2162 | 2024-10-31T00:18:40.233Z | INFO | SledAgent (InstanceManager): Booting oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd zone
file = illumos-utils/src/running_zone.rs:488
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
zone = oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2163 | 2024-10-31T00:18:40.401Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd") failed: Property not found. retry in 55.37347ms
file = illumos-utils/src/svc.rs:36
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
zone = oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2164 | 2024-10-31T00:18:40.470Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd") failed: Property not found. retry in 102.241067ms
file = illumos-utils/src/svc.rs:36
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
zone = oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2165 | 2024-10-31T00:18:40.587Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd") failed: Property not found. retry in 212.668707ms
file = illumos-utils/src/svc.rs:36
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
zone = oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2166 | 2024-10-31T00:18:40.813Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd") failed: Property not found. retry in 567.566986ms
file = illumos-utils/src/svc.rs:36
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
zone = oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2167 | 2024-10-31T00:18:41.391Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd") failed: Property not found. retry in 541.398697ms
file = illumos-utils/src/svc.rs:36
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
zone = oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2168 | 2024-10-31T00:18:41.946Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd") failed: Property not found. retry in 761.568912ms
file = illumos-utils/src/svc.rs:36
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
zone = oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2169 | 2024-10-31T00:18:42.719Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd") failed: Property not found. retry in 693.342252ms
file = illumos-utils/src/svc.rs:36
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
zone = oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2170 | 2024-10-31T00:18:43.422Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd") failed: Property not found. retry in 1.129316972s
file = illumos-utils/src/svc.rs:36
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
zone = oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2171 | 2024-10-31T00:18:44.563Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd") failed: Property not found. retry in 1.100795747s
file = illumos-utils/src/svc.rs:36
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
zone = oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2172 | 2024-10-31T00:18:45.693Z | INFO | SledAgent (InstanceManager): Started propolis in zone: oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
file = sled-agent/src/instance.rs:1787
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2173 | 2024-10-31T00:18:45.701Z | INFO | SledAgent (InstanceManager): Propolis SMF service is online
file = sled-agent/src/instance.rs:1796
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2174 | 2024-10-31T00:18:45.737Z | INFO | SledAgent (InstanceManager): Propolis HTTP server online
file = sled-agent/src/instance.rs:1820
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2175 | 2024-10-31T00:18:45.875Z | INFO | SledAgent (InstanceManager): result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })
file = sled-agent/src/instance.rs:932
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2176 | 2024-10-31T00:18:45.877Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-10-31T00:18:45.877147927Z }
file = sled-agent/src/instance.rs:780
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2177 | 2024-10-31T00:18:45.877Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:802
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
new_vmm_state = VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-10-31T00:18:45.877147927Z }
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2178 | 2024-10-31T00:18:45.878Z | INFO | SledAgent: IGW mapping changed; external IPs refreshed
file = sled-agent/src/sled_agent.rs:1215
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 2179 | 2024-10-31T00:18:45.878Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:700
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
state = SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-10-31T00:18:45.877147927Z }, migration_in: None, migration_out: None }
|
| 2180 | 2024-10-31T00:18:45.878Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 7088882
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:57528
req_id = f130461b-c29f-4505-8150-5a38f9caa3ed
response_code = 204
uri = /eip-gateways
|
| 2181 | 2024-10-31T00:18:45.890Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-10-31T00:18:45.890981608Z }
file = sled-agent/src/instance.rs:780
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2182 | 2024-10-31T00:18:45.891Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:802
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
new_vmm_state = VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-10-31T00:18:45.890981608Z }
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2183 | 2024-10-31T00:18:45.891Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:700
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
state = SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-10-31T00:18:45.890981608Z }, migration_in: None, migration_out: None }
|
| 2184 | 2024-10-31T00:18:45.954Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 399
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:57528
req_id = ea621160-486a-40da-808c-37e37d264fdd
response_code = 204
uri = /vpc-routes
|
| 2185 | 2024-10-31T00:18:46.344Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:50507
|
| 2186 | 2024-10-31T00:18:46.344Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 72
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:50507
req_id = 18a1be89-46ee-4fe1-8bad-51a108ceb0aa
response_code = 200
uri = /vpc-routes
|
| 2187 | 2024-10-31T00:18:46.350Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:36061
|
| 2188 | 2024-10-31T00:18:46.353Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 120
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:36061
req_id = 4859292e-4689-44db-93fe-d7366a5003fe
response_code = 200
uri = /vmms/8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd/state
|
| 2189 | 2024-10-31T00:18:46.406Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {487b7899-43a4-486a-bd8d-ca0047fd7478: {10.151.2.176: {1865c5a2-f061-4256-aaff-e92277867544}, 10.151.2.175: {1865c5a2-f061-4256-aaff-e92277867544}}} }
|
| 2190 | 2024-10-31T00:18:46.406Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 149
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:50507
req_id = 83cd911a-cda0-4f68-8947-7eaae6831573
response_code = 204
uri = /eip-gateways
|
| 2191 | 2024-10-31T00:18:46.424Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 234
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::c]:50507
req_id = a1f6231b-b2f5-4324-935a-21e93daebf17
response_code = 204
uri = /vpc-routes
|
| 2192 | 2024-10-31T00:18:46.645Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:50493
|
| 2193 | 2024-10-31T00:18:46.645Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 153
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:50493
req_id = 71633de2-86eb-4b0d-927c-33aa5f27180d
response_code = 200
uri = /v2p
|
| 2194 | 2024-10-31T00:18:46.701Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::c]:57958
|
| 2195 | 2024-10-31T00:18:46.701Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 2196 | 2024-10-31T00:18:46.737Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2197 | 2024-10-31T00:18:46.793Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 92247
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::c]:57958
req_id = 80b84384-0ab4-4e1b-ba41-7698a34b286c
response_code = 204
uri = /switch-ports
|
| 2198 | 2024-10-31T00:18:46.793Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 94
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::c]:57958
req_id = 6be128d9-160b-4870-9b44-70e6df19deba
response_code = 200
uri = /network-bootstore-config
|
| 2199 | 2024-10-31T00:18:47.162Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:36675
|
| 2200 | 2024-10-31T00:18:47.162Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 64
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:36675
req_id = 2598ff50-8aa9-4d52-b399-f84b46a34ea8
response_code = 200
uri = /vpc-routes
|
| 2201 | 2024-10-31T00:18:47.218Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:35834
|
| 2202 | 2024-10-31T00:18:47.220Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 118
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:35834
req_id = 0340a8a2-148a-47af-8c98-c52e549de8ee
response_code = 200
uri = /vmms/8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd/state
|
| 2203 | 2024-10-31T00:18:47.244Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:50219
|
| 2204 | 2024-10-31T00:18:47.247Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 89
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:50219
req_id = bdb1558c-3a1c-4557-8ae6-a4ae753e92ea
response_code = 200
uri = /vmms/8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd/state
|
| 2205 | 2024-10-31T00:18:47.292Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {487b7899-43a4-486a-bd8d-ca0047fd7478: {10.151.2.176: {1865c5a2-f061-4256-aaff-e92277867544}, 10.151.2.175: {1865c5a2-f061-4256-aaff-e92277867544}}} }
|
| 2206 | 2024-10-31T00:18:47.293Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 136
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:36675
req_id = 5e48216c-fc02-46c4-b3dd-113d77b8f093
response_code = 204
uri = /eip-gateways
|
| 2207 | 2024-10-31T00:18:47.311Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 300
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:36675
req_id = d94853d5-3210-42dc-821d-a527e73b2816
response_code = 204
uri = /vpc-routes
|
| 2208 | 2024-10-31T00:18:47.358Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:59534
|
| 2209 | 2024-10-31T00:18:47.358Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 168
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:59534
req_id = fbd5dc94-0d41-47a8-bea5-a92af51c404f
response_code = 200
uri = /v2p
|
| 2210 | 2024-10-31T00:18:47.359Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:56100
|
| 2211 | 2024-10-31T00:18:47.359Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 45
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:56100
req_id = 74121ff0-2389-4eb4-ac34-66b42ac28812
response_code = 200
uri = /vpc-routes
|
| 2212 | 2024-10-31T00:18:47.379Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:59859
|
| 2213 | 2024-10-31T00:18:47.379Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 2214 | 2024-10-31T00:18:47.422Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2215 | 2024-10-31T00:18:47.482Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 102725
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::b]:59859
req_id = ce1b0fb7-12a5-463f-97f3-3b2bf8de73e6
response_code = 204
uri = /switch-ports
|
| 2216 | 2024-10-31T00:18:47.482Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {487b7899-43a4-486a-bd8d-ca0047fd7478: {10.151.2.176: {1865c5a2-f061-4256-aaff-e92277867544}, 10.151.2.175: {1865c5a2-f061-4256-aaff-e92277867544}}} }
|
| 2217 | 2024-10-31T00:18:47.482Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 99
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:56100
req_id = d0c46165-48de-4b68-a3ca-6d1cc88e399d
response_code = 204
uri = /eip-gateways
|
| 2218 | 2024-10-31T00:18:47.482Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 90
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:59859
req_id = fed7f214-32ce-46a5-b9dd-deaaa4936627
response_code = 200
uri = /network-bootstore-config
|
| 2219 | 2024-10-31T00:18:47.544Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:53881
|
| 2220 | 2024-10-31T00:18:47.544Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 117
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:53881
req_id = 5736000e-5e12-444d-af46-9daaa89c5d88
response_code = 200
uri = /v2p
|
| 2221 | 2024-10-31T00:18:47.556Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 157
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:56100
req_id = 6df17865-3619-4fe0-afb6-19c63703e558
response_code = 204
uri = /vpc-routes
|
| 2222 | 2024-10-31T00:18:47.628Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:55082
|
| 2223 | 2024-10-31T00:18:47.628Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4081
|
| 2224 | 2024-10-31T00:18:47.668Z | INFO | SledAgent (ServiceManager): setting up port_qsfp0
file = sled-agent/src/services.rs:4105
|
| 2225 | 2024-10-31T00:18:47.722Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 94279
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::a]:55082
req_id = a5e3b867-eb29-470c-b4a4-c9a06c47186b
response_code = 204
uri = /switch-ports
|
| 2226 | 2024-10-31T00:18:47.723Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 99
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:55082
req_id = 0d96325e-9d70-4c68-a1e2-b93a92bfc817
response_code = 200
uri = /network-bootstore-config
|
| 2227 | 2024-10-31T00:18:56.259Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:49684
|
| 2228 | 2024-10-31T00:18:56.260Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 662
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:49684
req_id = 2b9ef96d-6a3d-487b-aea4-8dae7fa0434f
response_code = 200
uri = /vmms/8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd/state
|
| 2229 | 2024-10-31T00:18:56.260Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-10-31T00:18:56.260798915Z }
file = sled-agent/src/instance.rs:780
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2230 | 2024-10-31T00:18:56.260Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:802
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
new_vmm_state = VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-10-31T00:18:56.260798915Z }
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2231 | 2024-10-31T00:18:56.260Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:700
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
state = SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-10-31T00:18:56.260798915Z }, migration_in: None, migration_out: None }
|
| 2232 | 2024-10-31T00:18:56.269Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-10-31T00:18:56.269651066Z }
file = sled-agent/src/instance.rs:780
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2233 | 2024-10-31T00:18:56.269Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:802
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
new_vmm_state = VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-10-31T00:18:56.269651066Z }
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2234 | 2024-10-31T00:18:56.269Z | INFO | SledAgent (InstanceManager): terminating VMM that has exited
file = sled-agent/src/instance.rs:816
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2235 | 2024-10-31T00:18:56.269Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 2236 | 2024-10-31T00:18:56.269Z | WARN | SledAgent (kstat-sampler-worker): target removed with queued samples
file = oximeter/instruments/src/kstat/sampler.rs:602
id = 3209665516594333177
n_samples = 6
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 2237 | 2024-10-31T00:18:56.270Z | WARN | SledAgent (kstat-sampler-worker): target removed with queued samples
file = oximeter/instruments/src/kstat/sampler.rs:602
id = 2300910021289903676
n_samples = 6
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 2238 | 2024-10-31T00:18:56.272Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:451
|
| 2239 | 2024-10-31T00:18:56.273Z | INFO | SledAgent (ZoneBundler): creating zone bundle
context = ZoneBundleContext { storage_dirs: ["/pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/debug/bundle/zone", "/pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/debug/bundle/zone"], cause: TerminatedInstance, extra_log_dirs: ["/pool/ext/1d6671a7-5622-4671-b7bf-0a757330cf6b/crypt/debug", "/pool/ext/7dd5e852-297b-4341-9eef-72d5112891d5/crypt/debug", "/pool/ext/efb62c8a-6541-476c-8447-df5266c7fcb3/crypt/debug", "/pool/ext/ca55523e-99d5-46dd-9043-66e4f721f4aa/crypt/debug", "/pool/ext/67e5554d-50d7-4b6f-b9c3-723d8231ab86/crypt/debug", "/pool/ext/1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354/crypt/debug", "/pool/ext/a2173f6b-5437-4ef6-a016-ab4b5f89f5d8/crypt/debug", "/pool/ext/a536fa1e-3243-42d2-9446-aa31a45af0ee/crypt/debug", "/pool/ext/b43e5889-7a07-42ec-9599-254e55788daa/crypt/debug"] }
file = sled-agent/src/zone_bundle.rs:339
zone_name = oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2240 | 2024-10-31T00:18:56.894Z | INFO | SledAgent (ZoneBundler): finished zone bundle
file = sled-agent/src/zone_bundle.rs:1142
metadata = ZoneBundleMetadata { id: ZoneBundleId { zone_name: "oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd", bundle_id: 4d2b7470-6b2d-4389-a533-e45dc0315755 }, time_created: 2024-10-31T00:18:56.273540895Z, version: 0, cause: TerminatedInstance }
|
| 2241 | 2024-10-31T00:18:56.895Z | WARN | SledAgent (InstanceManager): Halting and removing zone: oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
file = sled-agent/src/instance.rs:1032
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2242 | 2024-10-31T00:18:58.623Z | INFO | SledAgent (InstanceManager): halt_and_remove_logged: Previous zone state: Running
file = illumos-utils/src/zone.rs:279
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2243 | 2024-10-31T00:18:58.640Z | INFO | SledAgent (PortManager): Removed route set for subnet
file = illumos-utils/src/opte/port_manager.rs:964
id = RouterId { vni: Vni(15018464), kind: System }
|
| 2244 | 2024-10-31T00:18:58.640Z | INFO | SledAgent (PortManager): Removed route set for subnet
file = illumos-utils/src/opte/port_manager.rs:964
id = RouterId { vni: Vni(15018464), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }
|
| 2245 | 2024-10-31T00:18:58.641Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:700
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
state = SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-10-31T00:18:58.641286306Z }, migration_in: None, migration_out: None }
|
| 2246 | 2024-10-31T00:18:58.666Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:700
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
state = SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-10-31T00:18:58.641286306Z }, migration_in: None, migration_out: None }
|
| 2247 | 2024-10-31T00:18:58.669Z | INFO | SledAgent (InstanceManager): State monitoring task complete
file = sled-agent/src/instance.rs:969
instance_id = 054c3697-a46c-4a3c-95ef-4a8d9a5537c2
propolis_id = 8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd
|
| 2248 | 2024-10-31T00:18:59.147Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:62218
|
| 2249 | 2024-10-31T00:18:59.147Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 55
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:62218
req_id = 144fd918-0181-4b22-ad54-528c836beb02
response_code = 200
uri = /vpc-routes
|
| 2250 | 2024-10-31T00:18:59.199Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 2251 | 2024-10-31T00:18:59.199Z | INFO | SledAgent: IGW mapping changed; external IPs refreshed
file = sled-agent/src/sled_agent.rs:1215
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
|
| 2252 | 2024-10-31T00:18:59.199Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 126
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:62218
req_id = 9d914043-02b6-4a97-911b-843da8bcffce
response_code = 204
uri = /eip-gateways
|
| 2253 | 2024-10-31T00:18:59.223Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:34214
|
| 2254 | 2024-10-31T00:18:59.223Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 105
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:34214
req_id = 388cf485-bf51-4f64-924e-4f3e91af3b8d
response_code = 200
uri = /v2p
|
| 2255 | 2024-10-31T00:18:59.232Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 146
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:62218
req_id = 40dcb312-b3ed-416f-88b9-af11b6b7416a
response_code = 204
uri = /vpc-routes
|
| 2256 | 2024-10-31T00:18:59.695Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:51499
|
| 2257 | 2024-10-31T00:18:59.695Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 179
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:51499
req_id = 01b625d9-3143-4d81-95c4-59ad71071bdd
response_code = 200
uri = /v2p
|
| 2258 | 2024-10-31T00:18:59.823Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:45699
|
| 2259 | 2024-10-31T00:18:59.823Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 70
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:45699
req_id = b466786c-273b-42ce-8305-a1b2ae27c8f4
response_code = 200
uri = /vpc-routes
|
| 2260 | 2024-10-31T00:18:59.838Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 2261 | 2024-10-31T00:18:59.838Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 71
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:45699
req_id = 9e26a65d-32d4-487d-aa90-fb2311cb2886
response_code = 204
uri = /eip-gateways
|
| 2262 | 2024-10-31T00:18:59.847Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 217
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:45699
req_id = 17092177-e219-40e9-bb94-876e043b3df9
response_code = 204
uri = /vpc-routes
|
| 2263 | 2024-10-31T00:19:00.519Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:61090
|
| 2264 | 2024-10-31T00:19:00.519Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 36
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::a]:61090
req_id = 0b9a78ec-8d0e-4756-b7b4-e4aac67bbd05
response_code = 200
uri = /vpc-routes
|
| 2265 | 2024-10-31T00:19:00.536Z | INFO | SledAgent: IGW mapping received
file = sled-agent/src/sled_agent.rs:1189
sled_id = f95bcc91-ac8f-446a-8a84-3ca7f7459fd7
values = ExternalIpGatewayMap { mappings: {} }
|
| 2266 | 2024-10-31T00:19:00.536Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 43
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:61090
req_id = c6b75697-3f69-44dc-8dc1-c28eb2af9713
response_code = 204
uri = /eip-gateways
|
| 2267 | 2024-10-31T00:19:00.545Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 185
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:61090
req_id = 31c22984-e404-4bcd-bf49-998bee81e852
response_code = 204
uri = /vpc-routes
|
| 2268 | 2024-10-31T00:19:00.944Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:62139
|
| 2269 | 2024-10-31T00:19:00.945Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 35
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:62139
req_id = c17cea2d-9bb0-4e6a-9c07-6f9d5716d315
response_code = 200
uri = /baseboard
|