| 1 | | [ Dec 19 00:37:47 Enabled. ] |
| 2 | | [ Dec 19 00:37:47 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] |
| 3 | | [ Dec 19 00:37:47 Method "start" exited with status 0. ] |
| 4 | | [ Dec 19 00:37:47 Rereading configuration. ] |
| 5 | | [ Dec 19 00:37:47 No 'refresh' method defined. Treating as :true. ] |
| 6 | | note: configured to log to "/dev/stdout" |
| 7 | 2024-12-19T00:37:47.955Z | INFO | SledAgent: Ensuring contract reaper thread
file = illumos-utils/src/running_zone.rs:113
|
| 8 | 2024-12-19T00:37:47.956Z | INFO | SledAgent (BootstrapAgentStartup): finding links ["net0", "net1"]
file = sled-agent/src/bootstrap/pre_server.rs:255
|
| 9 | 2024-12-19T00:37:48.008Z | INFO | SledAgent (BootstrapAgentStartup): Starting mg-ddm service
addr-objs = [AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]
file = sled-agent/src/bootstrap/pre_server.rs:262
|
| 10 | 2024-12-19T00:37:48.008Z | INFO | SledAgent (BootstrapAgentStartup): Importing mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:42
path = /opt/oxide/mg-ddm/pkg/ddm/manifest.xml
|
| 11 | 2024-12-19T00:37:48.123Z | INFO | SledAgent (BootstrapAgentStartup): Setting mg-ddm interfaces
file = sled-agent/src/bootstrap/maghemite.rs:50
interfaces = ("net0/ll" "net1/ll")
|
| 12 | 2024-12-19T00:37:48.132Z | INFO | SledAgent (BootstrapAgentStartup): Enabling mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:56
|
| 13 | 2024-12-19T00:37:48.140Z | INFO | SledAgent (BootstrapAgentStartup): Ensuring zfs key directory exists
file = sled-agent/src/bootstrap/pre_server.rs:272
path = /var/run/oxide/
|
| 14 | 2024-12-19T00:37:48.745Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }
|
| 15 | 2024-12-19T00:37:48.886Z | INFO | SledAgent: Starting KeyManager
file = sled-agent/src/long_running_tasks.rs:126
|
| 16 | 2024-12-19T00:37:48.886Z | INFO | SledAgent: Starting StorageManager
file = sled-agent/src/long_running_tasks.rs:138
|
| 17 | 2024-12-19T00:37:48.886Z | INFO | SledAgent: Starting StorageMonitor
file = sled-agent/src/long_running_tasks.rs:151
|
| 18 | 2024-12-19T00:37:48.886Z | INFO | SledAgent (DumpSetup-worker): DumpSetup poll loop started.
file = sled-agent/src/dump_setup.rs:520
|
| 19 | 2024-12-19T00:37:48.886Z | INFO | SledAgent: Starting HardwareManager
file = sled-agent/src/long_running_tasks.rs:173
nongimlet_observed_disks = []
sled_mode = Scrimlet { asic: SoftNpuZone }
|
| 20 | 2024-12-19T00:37:48.886Z | INFO | SledAgent (HardwareManager): Creating HardwareManager
file = sled-hardware/src/illumos/mod.rs:717
|
| 21 | 2024-12-19T00:37:48.890Z | INFO | SledAgent (HardwareManager): Generated i86pc baseboard Pc { identifier: "sock", model: "i86pc" }
file = sled-hardware/src/illumos/mod.rs:624
|
| 22 | 2024-12-19T00:37:48.890Z | WARN | SledAgent (HardwareManager): Device is not a Gimlet (i86pc), proceeding with null hardware view
file = sled-hardware/src/illumos/mod.rs:766
|
| 23 | 2024-12-19T00:37:48.890Z | INFO | SledAgent: Starting HardwareMonitor
file = sled-agent/src/long_running_tasks.rs:187
|
| 24 | 2024-12-19T00:37:48.890Z | INFO | SledAgent (HardwareManager): Monitoring for hardware updates
file = sled-hardware/src/illumos/mod.rs:825
|
| 25 | 2024-12-19T00:37:48.890Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/m2_0.vdev
|
| 26 | 2024-12-19T00:37:48.890Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
disks = {}
file = sled-agent/src/hardware_monitor.rs:271
underlay_network_info = None
|
| 27 | 2024-12-19T00:37:48.890Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:603
|
| 28 | 2024-12-19T00:37:48.890Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 29 | 2024-12-19T00:37:48.890Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
path = /scratch/m2_0.vdev
|
| 30 | 2024-12-19T00:37:48.902Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev
file = sled-hardware/src/disk.rs:367
|
| 31 | 2024-12-19T00:37:48.902Z | INFO | SledAgent (StorageResources): Formatting zpool with generated ID
file = sled-hardware/src/disk.rs:378
id = 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool)
|
| 32 | 2024-12-19T00:37:49.001Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:196
zpool = ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal }
|
| 33 | 2024-12-19T00:37:49.460Z | INFO | SledAgent (StorageManager): Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }
file = sled-storage/src/manager.rs:603
|
| 34 | 2024-12-19T00:37:49.460Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/m2_1.vdev
|
| 35 | 2024-12-19T00:37:49.460Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 36 | 2024-12-19T00:37:49.460Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 37 | 2024-12-19T00:37:49.460Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 38 | 2024-12-19T00:37:49.460Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:603
|
| 39 | 2024-12-19T00:37:49.460Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 40 | 2024-12-19T00:37:49.460Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 41 | 2024-12-19T00:37:49.460Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 42 | 2024-12-19T00:37:49.460Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
path = /scratch/m2_1.vdev
|
| 43 | 2024-12-19T00:37:49.469Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev
file = sled-hardware/src/disk.rs:367
|
| 44 | 2024-12-19T00:37:49.469Z | INFO | SledAgent (StorageResources): Formatting zpool with generated ID
file = sled-hardware/src/disk.rs:378
id = 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool)
|
| 45 | 2024-12-19T00:37:49.557Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:196
zpool = ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal }
|
| 46 | 2024-12-19T00:37:50.032Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_0.vdev
|
| 47 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 48 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 49 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 50 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:603
|
| 51 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 52 | 2024-12-19T00:37:50.032Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_1.vdev
|
| 53 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 54 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 55 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 56 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:603
|
| 57 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 58 | 2024-12-19T00:37:50.032Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_2.vdev
|
| 59 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:603
|
| 60 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 61 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 62 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 63 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 64 | 2024-12-19T00:37:50.032Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_3.vdev
|
| 65 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:603
|
| 66 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 67 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 68 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 69 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 70 | 2024-12-19T00:37:50.032Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_4.vdev
|
| 71 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 72 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 73 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 74 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:603
|
| 75 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 76 | 2024-12-19T00:37:50.032Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_5.vdev
|
| 77 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 78 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:603
|
| 79 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 80 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 81 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 82 | 2024-12-19T00:37:50.032Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_6.vdev
|
| 83 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 84 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 85 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 86 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:603
|
| 87 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 88 | 2024-12-19T00:37:50.032Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_7.vdev
|
| 89 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:603
|
| 90 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 91 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 92 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 93 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 94 | 2024-12-19T00:37:50.032Z | INFO | SledAgent: Upserting synthetic device to Storage Manager
file = sled-agent/src/long_running_tasks.rs:243
vdev = /scratch/u2_8.vdev
|
| 95 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageManager): Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }), tx: ... }
file = sled-storage/src/manager.rs:603
|
| 96 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageResources): Inserting disk
file = sled-storage/src/resources.rs:457
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 97 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 98 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 99 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 100 | 2024-12-19T00:37:50.032Z | INFO | SledAgent: Waiting for boot disk
file = sled-agent/src/long_running_tasks.rs:97
|
| 101 | 2024-12-19T00:37:50.032Z | INFO | SledAgent: Found boot disk DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
file = sled-agent/src/long_running_tasks.rs:99
|
| 102 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }: Unmanaged(Synthetic(RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 103 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 104 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 105 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 106 | 2024-12-19T00:37:50.032Z | INFO | SledAgent: Starting Bootstore
file = sled-agent/src/long_running_tasks.rs:211
|
| 107 | 2024-12-19T00:37:50.032Z | INFO | SledAgent (bootstore): No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 108 | 2024-12-19T00:37:50.033Z | INFO | SledAgent (bootstore): No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 109 | 2024-12-19T00:37:50.033Z | INFO | SledAgent (bootstore): No ledger found. Loading Fsm::State as Uninitialized
file = bootstore/src/schemes/v0/storage.rs:83
peer_id = pc-sock-i86pc
|
| 110 | 2024-12-19T00:37:50.033Z | INFO | SledAgent (bootstore): No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/cluster/bootstore-network-config.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 111 | 2024-12-19T00:37:50.033Z | INFO | SledAgent (bootstore): No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/cluster/bootstore-network-config.json
file = common/src/ledger.rs:188
peer_id = pc-sock-i86pc
|
| 112 | 2024-12-19T00:37:50.033Z | INFO | SledAgent (bootstore): No ledger found for network config
file = bootstore/src/schemes/v0/storage.rs:146
peer_id = pc-sock-i86pc
|
| 113 | 2024-12-19T00:37:50.033Z | INFO | SledAgent: Starting Bootstore DDMD poller
file = sled-agent/src/long_running_tasks.rs:216
|
| 114 | 2024-12-19T00:37:50.033Z | INFO | SledAgent: Starting ZoneBundler related tasks
file = sled-agent/src/long_running_tasks.rs:231
|
| 115 | 2024-12-19T00:37:50.043Z | INFO | SledAgent (ServiceManager): Creating ServiceManager
file = sled-agent/src/services.rs:736
|
| 116 | 2024-12-19T00:37:50.043Z | INFO | SledAgent (auto-cleanup-task): top of bundle cleanup loop
file = sled-agent/src/zone_bundle.rs:186
next_cleanup = Instant { tv_sec: 1394, tv_nsec: 796520085 }
time_to_next_cleanup = 599.999951209s
|
| 117 | 2024-12-19T00:37:50.043Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3947
|
| 118 | 2024-12-19T00:37:50.043Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 119 | 2024-12-19T00:37:50.043Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/sled-agent-request.json
file = common/src/ledger.rs:188
|
| 120 | 2024-12-19T00:37:50.046Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/sled-agent-request.json
file = common/src/ledger.rs:188
|
| 121 | 2024-12-19T00:37:50.047Z | INFO | SledAgent (dropshot (BootstrapAgent)): listening
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
|
| 122 | 2024-12-19T00:37:50.048Z | INFO | SledAgent (SledAgentSprocketsServer): Started listening
file = sled-agent/src/bootstrap/sprockets_server.rs:51
local_addr = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 123 | 2024-12-19T00:37:50.048Z | INFO | SledAgent (RSS): Injecting RSS configuration: RackInitializeRequest {
trust_quorum_peers: None,
bootstrap_discovery: OnlyOurs,
ntp_servers: [
"0.pool.ntp.org",
],
dns_servers: [
1.1.1.1,
9.9.9.9,
],
internal_services_ip_pool_ranges: [
V4(
Ipv4Range {
first: 10.151.2.164,
last: 10.151.2.173,
},
),
],
external_dns_ips: [
10.151.2.164,
10.151.2.165,
],
external_dns_zone_name: "oxide.test",
external_certificates: "<redacted>",
recovery_silo: RecoverySiloConfig {
silo_name: Name(
"recovery",
),
user_name: UserId(
"recovery",
),
user_password_hash: NewPasswordHash(
PasswordHashString {
string: "$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY",
encoding: B64,
},
),
},
rack_network_config: RackNetworkConfigV2 {
rack_subnet: Ipv6Net {
addr: fd00:1122:3344:100::,
width: 56,
},
infra_ip_first: 10.151.2.174,
infra_ip_last: 10.151.2.174,
ports: [
PortConfigV2 {
routes: [
RouteConfig {
destination: V4(
Ipv4Net {
addr: 0.0.0.0,
width: 0,
},
),
nexthop: 10.151.2.100,
vlan_id: None,
rib_priority: None,
},
],
addresses: [
UplinkAddressConfig {
address: V4(
Ipv4Net {
addr: 10.151.2.174,
width: 24,
},
),
vlan_id: None,
},
],
switch: Switch0,
port: "qsfp0",
uplink_port_speed: Speed40G,
uplink_port_fec: Some(
None,
),
bgp_peers: [],
autoneg: false,
lldp: None,
tx_eq: None,
},
],
bgp: [],
bfd: [],
},
allowed_source_ips: Any,
}
file = sled-agent/src/rack_setup/service.rs:1203
|
| 124 | 2024-12-19T00:37:50.048Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = internal-dns/resolver/src/resolver.rs:111
|
| 125 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 126 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 127 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (RSS): No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/rss-started.marker
file = common/src/ledger.rs:188
|
| 128 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (RSS): No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/rss-started.marker
file = common/src/ledger.rs:188
|
| 129 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (RSS): No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/rss-plan-completed.marker
file = common/src/ledger.rs:188
|
| 130 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (RSS): No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/rss-plan-completed.marker
file = common/src/ledger.rs:188
|
| 131 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (RSS): RSS not previously run. Creating plans.
file = sled-agent/src/rack_setup/service.rs:1252
|
| 132 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/.rss-started.marker.tmp
file = common/src/ledger.rs:199
|
| 133 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/.rss-started.marker.tmp
file = common/src/ledger.rs:199
|
| 134 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (RSS): Creating new allocation plan
file = sled-agent/src/rack_setup/service.rs:1284
|
| 135 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (RSS): Serializing plan
file = sled-agent/src/rack_setup/plan/sled.rs:66
|
| 136 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (RSS): Creating plan for the sled at fdb0:18c0:4d0d:a02a::1
file = sled-agent/src/rack_setup/plan/sled.rs:39
|
| 137 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (RSS): Writing Rack Network Configuration to bootstore
file = sled-agent/src/rack_setup/service.rs:1316
|
| 138 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 1, current_generation: 0
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
| 139 | 2024-12-19T00:37:50.049Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 140 | 2024-12-19T00:37:50.050Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:199
peer_id = pc-sock-i86pc
|
| 141 | 2024-12-19T00:37:50.050Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 1
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-sock-i86pc
|
| 142 | 2024-12-19T00:37:50.050Z | INFO | SledAgent (BootstrapAgentRssHandler): Received initialization request from RSS
file = sled-agent/src/bootstrap/rss_handle.rs:232
request = StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: e358e662-7828-4f7f-85a4-21e089483ee6 (sled), rack_id: ec268bea-b1b9-4b27-9476-6ec4bc3257c8, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 143 | 2024-12-19T00:37:50.050Z | INFO | SledAgent (ServiceManager): Enabling switch zone (new)
file = sled-agent/src/services.rs:4244
|
| 144 | 2024-12-19T00:37:50.050Z | INFO | SledAgent (ServiceManager): Starting switch zone
file = sled-agent/src/services.rs:4644
|
| 145 | 2024-12-19T00:37:50.055Z | INFO | SledAgent (SledAgentSprocketsClient): Loaded keys and certs
BootstrapAgentClient = [fdb0:18c0:4d0d:a02a::1]:12346
file = /home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50
|
| 146 | 2024-12-19T00:37:50.061Z | INFO | SledAgent (SledAgentSprocketsClient): Certificate chain verified successfully
BootstrapAgentClient = [fdb0:18c0:4d0d:a02a::1]:12346
file = /home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307
|
| 147 | 2024-12-19T00:37:50.061Z | INFO | SledAgent (SledAgentSprocketsClient): Signature verified successfully
BootstrapAgentClient = [fdb0:18c0:4d0d:a02a::1]:12346
file = /home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376
|
| 148 | 2024-12-19T00:37:50.065Z | INFO | SledAgent (SledAgentSprocketsServer): Certificate chain verified successfully
file = /home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307
|
| 149 | 2024-12-19T00:37:50.065Z | INFO | SledAgent (SledAgentSprocketsServer): Signature verified successfully
file = /home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376
|
| 150 | 2024-12-19T00:37:50.065Z | INFO | SledAgent (SledAgentSprocketsServer): Accepted connection
file = sled-agent/src/bootstrap/sprockets_server.rs:74
remote_addr = [fdb0:18c0:4d0d:a02a::1]:42770
|
| 151 | 2024-12-19T00:37:50.065Z | INFO | SledAgent (SledAgentMain): Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: e358e662-7828-4f7f-85a4-21e089483ee6 (sled), rack_id: ec268bea-b1b9-4b27-9476-6ec4bc3257c8, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }
file = sled-agent/src/bootstrap/server.rs:362
|
| 152 | 2024-12-19T00:37:50.065Z | INFO | SledAgent (SledAgentMain): KeyManager: using hardcoded secret retriever
file = sled-agent/src/bootstrap/server.rs:377
|
| 153 | 2024-12-19T00:37:50.065Z | INFO | SledAgent: setting up sled agent server
file = sled-agent/src/server.rs:44
|
| 154 | 2024-12-19T00:37:50.065Z | INFO | SledAgent (StorageManager): Received KeyManagerReady
file = sled-storage/src/manager.rs:603
|
| 155 | 2024-12-19T00:37:50.065Z | INFO | SledAgent (StorageManager): Loading storage config
file = sled-storage/src/manager.rs:809
|
| 156 | 2024-12-19T00:37:50.065Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = internal-dns/resolver/src/resolver.rs:111
|
| 157 | 2024-12-19T00:37:50.065Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }
|
| 158 | 2024-12-19T00:37:50.065Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = load_disks_ledger
|
| 159 | 2024-12-19T00:37:50.065Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = load_disks_ledger
|
| 160 | 2024-12-19T00:37:50.065Z | INFO | SledAgent (StorageManager): No ledger of physical disks exists
file = sled-storage/src/manager.rs:798
|
| 161 | 2024-12-19T00:37:50.065Z | INFO | SledAgent (StorageManager): KeyManager ready, but no ledger detected
file = sled-storage/src/manager.rs:826
|
| 162 | 2024-12-19T00:37:50.065Z | INFO | SledAgent (StorageManager): automatically managing disks
file = sled-storage/src/manager.rs:592
|
| 163 | 2024-12-19T00:37:50.066Z | INFO | SledAgent (StorageResources): Synchronizing disk managment
file = sled-storage/src/resources.rs:302
|
| 164 | 2024-12-19T00:37:50.066Z | INFO | SledAgent (StorageManager): Successfully synchronized disks without error
file = sled-storage/src/manager.rs:697
result = DisksManagementResult { status: [] }
|
| 165 | 2024-12-19T00:37:50.080Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 166 | 2024-12-19T00:37:50.101Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:325
|
| 167 | 2024-12-19T00:37:50.118Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:363
|
| 168 | 2024-12-19T00:37:50.129Z | INFO | SledAgent: SledAgent::new(..) starting
file = sled-agent/src/sled_agent.rs:415
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 169 | 2024-12-19T00:37:50.129Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 170 | 2024-12-19T00:37:50.129Z | INFO | SledAgent: Requested swap device of size 64 GiB
file = sled-agent/src/sled_agent.rs:427
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 171 | 2024-12-19T00:37:50.129Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 172 | 2024-12-19T00:37:50.132Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 101.304741ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "05ee4050-057d-4f62-b7bb-420f3da0890f", "content-length": "124", "date": "Thu, 19 Dec 2024 00:37:50 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "05ee4050-057d-4f62-b7bb-420f3da0890f" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 173 | 2024-12-19T00:37:50.143Z | INFO | SledAgent: attempting to create encrypted zvol: name="oxi_484faba5-5bf0-4a1e-b697-76cf9fed3ca8/swap", size_gb=64
file = sled-agent/src/swap_device.rs:153
|
| 174 | 2024-12-19T00:37:50.166Z | INFO | SledAgent: successfully created encrypted zvol: name="oxi_484faba5-5bf0-4a1e-b697-76cf9fed3ca8/swap", size_gb=64
file = sled-agent/src/swap_device.rs:249
|
| 175 | 2024-12-19T00:37:50.166Z | INFO | SledAgent: adding swap device: swapname="/dev/zvol/dsk/oxi_484faba5-5bf0-4a1e-b697-76cf9fed3ca8/swap"
file = sled-agent/src/swap_device.rs:96
|
| 176 | 2024-12-19T00:37:50.171Z | INFO | SledAgent: Mounting backing filesystems
file = sled-agent/src/sled_agent.rs:442
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 177 | 2024-12-19T00:37:50.171Z | INFO | SledAgent (BackingFs): Processing fmd
file = sled-agent/src/backing_fs.rs:128
|
| 178 | 2024-12-19T00:37:50.171Z | INFO | SledAgent (BackingFs): Ensuring dataset oxi_484faba5-5bf0-4a1e-b697-76cf9fed3ca8/backing/fmd
file = sled-agent/src/backing_fs.rs:138
|
| 179 | 2024-12-19T00:37:50.235Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 180 | 2024-12-19T00:37:50.235Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 169.216422ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6e76b941-3a40-4969-8737-1535b7b0f215", "content-length": "124", "date": "Thu, 19 Dec 2024 00:37:50 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6e76b941-3a40-4969-8737-1535b7b0f215" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 181 | 2024-12-19T00:37:50.257Z | INFO | SledAgent (BackingFs): Stopping service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:173
|
| 182 | 2024-12-19T00:37:50.406Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 183 | 2024-12-19T00:37:50.406Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 198.707334ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6d7a68d1-cb4d-460f-87e3-e3b62413fd2d", "content-length": "124", "date": "Thu, 19 Dec 2024 00:37:50 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6d7a68d1-cb4d-460f-87e3-e3b62413fd2d" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 184 | 2024-12-19T00:37:50.476Z | INFO | SledAgent (BackingFs): Mounting oxi_484faba5-5bf0-4a1e-b697-76cf9fed3ca8/backing/fmd on /var/fm/fmd
file = sled-agent/src/backing_fs.rs:181
|
| 185 | 2024-12-19T00:37:50.488Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/rsrc
file = sled-agent/src/backing_fs.rs:189
|
| 186 | 2024-12-19T00:37:50.488Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/ckpt
file = sled-agent/src/backing_fs.rs:189
|
| 187 | 2024-12-19T00:37:50.488Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/xprt
file = sled-agent/src/backing_fs.rs:189
|
| 188 | 2024-12-19T00:37:50.488Z | INFO | SledAgent (BackingFs): Starting service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:195
|
| 189 | 2024-12-19T00:37:50.606Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 190 | 2024-12-19T00:37:50.607Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 161.054176ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "fc5a6e38-fa6e-4686-b9f0-2602c189a13a", "content-length": "124", "date": "Thu, 19 Dec 2024 00:37:50 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "fc5a6e38-fa6e-4686-b9f0-2602c189a13a" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 191 | 2024-12-19T00:37:50.769Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 192 | 2024-12-19T00:37:50.770Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 160.466284ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6d926f56-fd36-4b17-bacb-df6b3dbe3790", "content-length": "124", "date": "Thu, 19 Dec 2024 00:37:50 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6d926f56-fd36-4b17-bacb-df6b3dbe3790" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 193 | 2024-12-19T00:37:50.931Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 194 | 2024-12-19T00:37:50.931Z | INFO | SledAgent (BootstrapAgentStartup): Failed enable ddm stats (will retry after 373.024171ms)
DdmAdminClient = [::1]:8000
err = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "97a79d25-f959-40ee-9426-1deb77a97a5f", "content-length": "124", "date": "Thu, 19 Dec 2024 00:37:50 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "97a79d25-f959-40ee-9426-1deb77a97a5f" }
file = clients/ddm-admin-client/src/lib.rs:177
|
| 195 | 2024-12-19T00:37:51.224Z | INFO | SledAgent: using '[AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]' as data links for xde driver
file = illumos-utils/src/opte/illumos.rs:93
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 196 | 2024-12-19T00:37:51.230Z | INFO | SledAgent (dropshot): listening
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327
local_addr = [fd00:1122:3344:101::1]:42252
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 197 | 2024-12-19T00:37:51.230Z | INFO | SledAgent (internal-dns-resolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = internal-dns/resolver/src/resolver.rs:111
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 198 | 2024-12-19T00:37:51.230Z | INFO | SledAgent (producer-server): starting oximeter metric producer server
address = [fd00:1122:3344:101::1]:42252
file = oximeter/producer/src/lib.rs:279
interval = 30s
producer_id = e358e662-7828-4f7f-85a4-21e089483ee6
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 199 | 2024-12-19T00:37:51.231Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 180.269804ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:387
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 200 | 2024-12-19T00:37:51.232Z | INFO | SledAgent (VmmReservoirManager): 50% of 102952026112 physical ram = 51476013056 bytes)
file = sled-agent/src/vmm_reservoir.rs:267
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 201 | 2024-12-19T00:37:51.232Z | INFO | SledAgent (VmmReservoirManager): Setting reservoir size to 49090 MiB bytes
file = sled-agent/src/vmm_reservoir.rs:275
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 202 | 2024-12-19T00:37:51.232Z | INFO | SledAgent (ServiceManager): sled agent started
file = sled-agent/src/services.rs:920
underlay_address = fd00:1122:3344:101::1
|
| 203 | 2024-12-19T00:37:51.232Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = internal-dns/resolver/src/resolver.rs:111
|
| 204 | 2024-12-19T00:37:51.305Z | INFO | SledAgent (BootstrapAgentStartup): Enabling ddm stats
DdmAdminClient = [::1]:8000
addr = fd00:1122:3344:101::1
dns_servers = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]
file = clients/ddm-admin-client/src/lib.rs:163
|
| 205 | 2024-12-19T00:37:51.413Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 257.260767ms
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:387
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 206 | 2024-12-19T00:37:51.672Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 1.307208878s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:387
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 207 | 2024-12-19T00:37:52.980Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 2.37311918s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:387
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 208 | 2024-12-19T00:37:53.738Z | INFO | SledAgent (ServiceManager): Setting up dendrite service
file = sled-agent/src/services.rs:2668
|
| 209 | 2024-12-19T00:37:53.738Z | INFO | SledAgent (ServiceManager): Setting up lldpd service
file = sled-agent/src/services.rs:2913
|
| 210 | 2024-12-19T00:37:53.738Z | INFO | SledAgent (ServiceManager): Setting up MGS service
file = sled-agent/src/services.rs:2538
|
| 211 | 2024-12-19T00:37:53.738Z | INFO | SledAgent (ServiceManager): Setting up wicketd service
file = sled-agent/src/services.rs:2585
|
| 212 | 2024-12-19T00:37:53.738Z | INFO | SledAgent (ServiceManager): Setting up mgd service
file = sled-agent/src/services.rs:2994
|
| 213 | 2024-12-19T00:37:53.738Z | INFO | SledAgent (ServiceManager): Setting up mg-ddm service
file = sled-agent/src/services.rs:3039
|
| 214 | 2024-12-19T00:37:53.740Z | INFO | SledAgent (ServiceManager): Setting up tfport service
file = sled-agent/src/services.rs:2838
|
| 215 | 2024-12-19T00:37:53.742Z | INFO | SledAgent (ServiceManager): Setting up Simulated SP service
file = sled-agent/src/services.rs:2579
|
| 216 | 2024-12-19T00:37:53.742Z | INFO | SledAgent (ServiceManager): Profile for oxz_switch:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService0'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='::1'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/mgs">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="address" value='[::1]:12225'/>
<propval type="astring" name="id" value='440f5054-10a1-4997-be16-da46ecf38c93'/>
<propval type="astring" name="rack_id" value='ec268bea-b1b9-4b27-9476-6ec4bc3257c8'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/wicketd">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="address" value='[::1]:12226'/>
<propval type="astring" name="artifact-address" value='[fdb0:18c0:4d0d:a02a::2]:12227'/>
<propval type="astring" name="baseboard-file" value='/opt/oxide/baseboard.json'/>
<propval type="astring" name="mgs-address" value='[::1]:12225'/>
<propval type="astring" name="nexus-proxy-address" value='[::]:12229'/>
<propval type="astring" name="rack-subnet" value='fd00:1122:3344::'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/switch_zone_setup">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="gz_local_link_addr" value='fe80::8:20ff:fe4c:e927'/>
<property type="astring" name="link_local_links">
<astring_list>
<value_node value="tfportrear0_0" />
<value_node value="oxBootstrap0" />
</astring_list>
</property>
<propval type="astring" name="bootstrap_addr" value='fdb0:18c0:4d0d:a02a::2'/>
<propval type="astring" name="bootstrap_vnic" value='oxBootstrap0'/>
<propval type="astring" name="baseboard_info" value='{"type":"pc","identifier":"sock","model":"i86pc"}'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/dendrite">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="rack_id" value='ec268bea-b1b9-4b27-9476-6ec4bc3257c8'/>
<propval type="astring" name="sled_id" value='e358e662-7828-4f7f-85a4-21e089483ee6'/>
<propval type="astring" name="sled_model" value='i86pc'/>
<propval type="astring" name="sled_serial" value='sock'/>
<propval type="astring" name="sled_revision" value='0'/>
<propval type="astring" name="address" value='[::1]:12224'/>
<propval type="astring" name="front_ports" value='1'/>
<propval type="astring" name="rear_ports" value='1'/>
<propval type="astring" name="port_config" value='/opt/oxide/dendrite/misc/softnpu_single_sled_config.toml'/>
<propval type="astring" name="mgmt" value='uds'/>
<propval type="astring" name="uds_path" value='/opt/softnpu/stuff'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/tfport">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="host" value='[::1]'/>
<propval type="astring" name="port" value='12224'/>
<propval type="astring" name="flags" value='--sync-only'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/lldpd">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="board_rev" value='softnpu_front_1_rear_1'/>
<propval type="astring" name="scrimlet_id" value='sock'/>
<propval type="astring" name="scrimlet_model" value='i86pc'/>
<propval type="astring" name="address" value='[::1]:12230'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/pumpkind">
</service>
<service version="1" type="service" name="oxide/mgd">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="sled_uuid" value='e358e662-7828-4f7f-85a4-21e089483ee6'/>
<propval type="astring" name="rack_uuid" value='ec268bea-b1b9-4b27-9476-6ec4bc3257c8'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/mg-ddm">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="mode" value='transit'/>
<propval type="astring" name="dendrite" value='true'/>
<propval type="astring" name="sled_uuid" value='e358e662-7828-4f7f-85a4-21e089483ee6'/>
<propval type="astring" name="rack_uuid" value='ec268bea-b1b9-4b27-9476-6ec4bc3257c8'/>
<propval type="astring" name="interfaces" value='tfportrear0_0/ll'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/uplink">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 217 | 2024-12-19T00:37:53.742Z | INFO | SledAgent (ServiceManager): Booting oxz_switch zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_switch
|
| 218 | 2024-12-19T00:37:54.024Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 63.176158ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 219 | 2024-12-19T00:37:54.235Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 141.165816ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 220 | 2024-12-19T00:37:54.422Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 198.987817ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 221 | 2024-12-19T00:37:54.769Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 450.580995ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 222 | 2024-12-19T00:37:55.362Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 886.081334ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 223 | 2024-12-19T00:37:55.362Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 3.134620657s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:387
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 224 | 2024-12-19T00:37:56.368Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 754.348345ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 225 | 2024-12-19T00:37:57.230Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 927.064474ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 226 | 2024-12-19T00:37:58.370Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 1.007630281s
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 227 | 2024-12-19T00:37:58.499Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 5.180234554s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:387
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 228 | 2024-12-19T00:37:59.902Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e358e662-7828-4f7f-85a4-21e089483ee6)
file = sled-agent/src/nexus.rs:527
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 229 | 2024-12-19T00:37:59.932Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 230 | 2024-12-19T00:37:59.932Z | INFO | SledAgent (dropshot (Repo Depot)): listening
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327
local_addr = [fd00:1122:3344:101::1]:12348
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 231 | 2024-12-19T00:37:59.933Z | INFO | SledAgent (dropshot (SledAgent)): listening
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327
local_addr = [fd00:1122:3344:101::1]:12345
|
| 232 | 2024-12-19T00:37:59.933Z | INFO | SledAgent (SledAgentMain): Sled Agent loaded; recording configuration
file = sled-agent/src/bootstrap/server.rs:432
|
| 233 | 2024-12-19T00:37:59.933Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 234 | 2024-12-19T00:37:59.933Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:199
|
| 235 | 2024-12-19T00:37:59.934Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:199
|
| 236 | 2024-12-19T00:37:59.934Z | INFO | SledAgent (HardwareMonitor): Sled Agent Started
file = sled-agent/src/hardware_monitor.rs:138
|
| 237 | 2024-12-19T00:37:59.934Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
disks = {}
file = sled-agent/src/hardware_monitor.rs:271
underlay_network_info = Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: "qsfp0", uplink_port_speed: Speed40G, uplink_port_fec: Some(None), bgp_peers: [], autoneg: false, lldp: None, tx_eq: None }], bgp: [], bfd: [] })))
|
| 238 | 2024-12-19T00:37:59.934Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3947
|
| 239 | 2024-12-19T00:37:59.934Z | INFO | SledAgent (SledAgentSprocketsServer): Connection closed
file = sled-agent/src/bootstrap/sprockets_server.rs:81
remote_addr = [fdb0:18c0:4d0d:a02a::1]:42770
|
| 240 | 2024-12-19T00:38:00.125Z | INFO | SledAgent (BootstrapAgentRssHandler): Peer agent initialized
file = sled-agent/src/bootstrap/rss_handle.rs:117
peer = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 241 | 2024-12-19T00:38:00.125Z | INFO | SledAgent (BootstrapAgentRssHandler): Initialized sled agent
file = sled-agent/src/bootstrap/rss_handle.rs:252
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 242 | 2024-12-19T00:38:00.131Z | INFO | SledAgent (ServiceManager): Re-enabling running switch zone (new address)
file = sled-agent/src/services.rs:4268
new = [fd00:1122:3344:101::2, ::1]
old = [::1]
|
| 243 | 2024-12-19T00:38:00.131Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists
file = sled-agent/src/services.rs:4284
|
| 244 | 2024-12-19T00:38:00.131Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:534
zone = oxz_switch
|
| 245 | 2024-12-19T00:38:00.400Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:65124
|
| 246 | 2024-12-19T00:38:00.400Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 247 | 2024-12-19T00:38:00.400Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 248 | 2024-12-19T00:38:00.400Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 249 | 2024-12-19T00:38:00.400Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 250 | 2024-12-19T00:38:00.400Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 305
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:65124
req_id = cf1baf1f-c252-4e2a-9664-99d270bad7a3
response_code = 200
uri = /inventory
|
| 251 | 2024-12-19T00:38:00.431Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:48883
|
| 252 | 2024-12-19T00:38:00.431Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 51
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:48883
req_id = f0c782f6-ce4b-4da0-91fe-819aefe98a5c
response_code = 200
uri = /sled-role
|
| 253 | 2024-12-19T00:38:00.461Z | INFO | SledAgent (RSS): attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, id: 34585501-77f9-46aa-bbe8-ab9978160629 (physical_disk), pool_id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, id: 32944af9-f1bc-460a-8b1f-0f379977d412 (physical_disk), pool_id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, id: 5339a28f-8415-4ebc-975f-7a8352cc53a6 (physical_disk), pool_id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, id: 21cce821-1fd9-4b57-9e1b-77489c0dba86 (physical_disk), pool_id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, id: d6ad0783-9d78-40d5-9cf3-2c223ca50cbf (physical_disk), pool_id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, id: 3735fb39-3079-43bb-9c04-9ad82213b986 (physical_disk), pool_id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, id: abda9141-7cf7-44e6-addd-c2b3f5051e7d (physical_disk), pool_id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, id: 2dbaf12a-2a59-4be4-9274-44f1a0e5842e (physical_disk), pool_id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, id: be669f5b-7fba-4429-9388-191e847a9125 (physical_disk), pool_id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool) }] }
file = sled-agent/src/rack_setup/service.rs:385
sled_address = [fd00:1122:3344:101::1]:12345
|
| 254 | 2024-12-19T00:38:00.461Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:45008
|
| 255 | 2024-12-19T00:38:00.461Z | INFO | SledAgent: physical disks ensure
file = sled-agent/src/sled_agent.rs:896
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 256 | 2024-12-19T00:38:00.461Z | INFO | SledAgent (StorageManager): Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, id: 34585501-77f9-46aa-bbe8-ab9978160629 (physical_disk), pool_id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, id: 32944af9-f1bc-460a-8b1f-0f379977d412 (physical_disk), pool_id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, id: 5339a28f-8415-4ebc-975f-7a8352cc53a6 (physical_disk), pool_id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, id: 21cce821-1fd9-4b57-9e1b-77489c0dba86 (physical_disk), pool_id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, id: d6ad0783-9d78-40d5-9cf3-2c223ca50cbf (physical_disk), pool_id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, id: 3735fb39-3079-43bb-9c04-9ad82213b986 (physical_disk), pool_id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, id: abda9141-7cf7-44e6-addd-c2b3f5051e7d (physical_disk), pool_id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, id: 2dbaf12a-2a59-4be4-9274-44f1a0e5842e (physical_disk), pool_id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, id: be669f5b-7fba-4429-9388-191e847a9125 (physical_disk), pool_id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool) }] }, tx: ... }
file = sled-storage/src/manager.rs:603
|
| 257 | 2024-12-19T00:38:00.461Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = omicron_physical_disks_ensure
|
| 258 | 2024-12-19T00:38:00.461Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/omicron-physical-disks.json
file = common/src/ledger.rs:188
request = omicron_physical_disks_ensure
|
| 259 | 2024-12-19T00:38:00.461Z | INFO | SledAgent (StorageManager): No previously-stored 'requested disks', creating new ledger
file = sled-storage/src/manager.rs:1171
request = omicron_physical_disks_ensure
|
| 260 | 2024-12-19T00:38:00.461Z | INFO | SledAgent (StorageResources): Synchronizing disk managment
file = sled-storage/src/resources.rs:302
|
| 261 | 2024-12-19T00:38:00.461Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:342
|
| 262 | 2024-12-19T00:38:00.461Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 263 | 2024-12-19T00:38:00.461Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
path = /scratch/u2_0.vdev
|
| 264 | 2024-12-19T00:38:00.606Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev
file = sled-hardware/src/disk.rs:367
|
| 265 | 2024-12-19T00:38:00.606Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool)
|
| 266 | 2024-12-19T00:38:01.052Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)
file = sled-agent/src/probe_manager.rs:231
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 267 | 2024-12-19T00:38:01.265Z | INFO | SledAgent (VmmReservoirManager): Finished setting reservoir size to 49090 MiB bytes
file = sled-agent/src/vmm_reservoir.rs:279
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 268 | 2024-12-19T00:38:01.266Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 14076285699142498485
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 269 | 2024-12-19T00:38:01.267Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 15791617063128661124
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 270 | 2024-12-19T00:38:01.312Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:196
zpool = ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }
|
| 271 | 2024-12-19T00:38:01.319Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/dataset.rs:240
|
| 272 | 2024-12-19T00:38:01.319Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:242
|
| 273 | 2024-12-19T00:38:01.319Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:246
|
| 274 | 2024-12-19T00:38:01.319Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
epoch = 0
file = sled-storage/src/dataset.rs:248
|
| 275 | 2024-12-19T00:38:01.319Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 276 | 2024-12-19T00:38:01.320Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 277 | 2024-12-19T00:38:01.320Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:260
|
| 278 | 2024-12-19T00:38:01.355Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 279 | 2024-12-19T00:38:01.363Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone
file = sled-storage/src/dataset.rs:303
|
| 280 | 2024-12-19T00:38:01.534Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_1a77147c-c146-4e56-8066-57b7b2b29250
file = sled-storage/src/dataset.rs:389
|
| 281 | 2024-12-19T00:38:01.541Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:440
|
| 282 | 2024-12-19T00:38:01.541Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
file = sled-storage/src/resources.rs:357
|
| 283 | 2024-12-19T00:38:01.541Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:342
|
| 284 | 2024-12-19T00:38:01.541Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 285 | 2024-12-19T00:38:01.541Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
path = /scratch/u2_1.vdev
|
| 286 | 2024-12-19T00:38:01.552Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev
file = sled-hardware/src/disk.rs:367
|
| 287 | 2024-12-19T00:38:01.552Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool)
|
| 288 | 2024-12-19T00:38:01.647Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:196
zpool = ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }
|
| 289 | 2024-12-19T00:38:01.654Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/dataset.rs:240
|
| 290 | 2024-12-19T00:38:01.655Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:242
|
| 291 | 2024-12-19T00:38:01.655Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:246
|
| 292 | 2024-12-19T00:38:01.655Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
epoch = 0
file = sled-storage/src/dataset.rs:248
|
| 293 | 2024-12-19T00:38:01.655Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 294 | 2024-12-19T00:38:01.655Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 295 | 2024-12-19T00:38:01.655Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:260
|
| 296 | 2024-12-19T00:38:01.692Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 297 | 2024-12-19T00:38:01.700Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone
file = sled-storage/src/dataset.rs:303
|
| 298 | 2024-12-19T00:38:01.869Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c
file = sled-storage/src/dataset.rs:389
|
| 299 | 2024-12-19T00:38:01.877Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:440
|
| 300 | 2024-12-19T00:38:01.877Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
file = sled-storage/src/resources.rs:357
|
| 301 | 2024-12-19T00:38:01.877Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:342
|
| 302 | 2024-12-19T00:38:01.877Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 303 | 2024-12-19T00:38:01.877Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
path = /scratch/u2_2.vdev
|
| 304 | 2024-12-19T00:38:01.886Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev
file = sled-hardware/src/disk.rs:367
|
| 305 | 2024-12-19T00:38:01.886Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool)
|
| 306 | 2024-12-19T00:38:01.983Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/dataset.rs:196
zpool = ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }
|
| 307 | 2024-12-19T00:38:01.991Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/dataset.rs:240
|
| 308 | 2024-12-19T00:38:01.991Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:242
|
| 309 | 2024-12-19T00:38:01.991Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:246
|
| 310 | 2024-12-19T00:38:01.991Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
epoch = 0
file = sled-storage/src/dataset.rs:248
|
| 311 | 2024-12-19T00:38:01.991Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 312 | 2024-12-19T00:38:01.991Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 313 | 2024-12-19T00:38:01.991Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:260
|
| 314 | 2024-12-19T00:38:02.026Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 315 | 2024-12-19T00:38:02.035Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone
file = sled-storage/src/dataset.rs:303
|
| 316 | 2024-12-19T00:38:02.054Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)
file = sled-agent/src/probe_manager.rs:231
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 317 | 2024-12-19T00:38:02.213Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7
file = sled-storage/src/dataset.rs:389
|
| 318 | 2024-12-19T00:38:02.220Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:440
|
| 319 | 2024-12-19T00:38:02.220Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
file = sled-storage/src/resources.rs:357
|
| 320 | 2024-12-19T00:38:02.220Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:342
|
| 321 | 2024-12-19T00:38:02.220Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 322 | 2024-12-19T00:38:02.220Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
path = /scratch/u2_3.vdev
|
| 323 | 2024-12-19T00:38:02.230Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev
file = sled-hardware/src/disk.rs:367
|
| 324 | 2024-12-19T00:38:02.230Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool)
|
| 325 | 2024-12-19T00:38:02.327Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/dataset.rs:196
zpool = ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }
|
| 326 | 2024-12-19T00:38:02.334Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/dataset.rs:240
|
| 327 | 2024-12-19T00:38:02.334Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:242
|
| 328 | 2024-12-19T00:38:02.334Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:246
|
| 329 | 2024-12-19T00:38:02.334Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
epoch = 0
file = sled-storage/src/dataset.rs:248
|
| 330 | 2024-12-19T00:38:02.334Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 331 | 2024-12-19T00:38:02.335Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 332 | 2024-12-19T00:38:02.335Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:260
|
| 333 | 2024-12-19T00:38:02.369Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 334 | 2024-12-19T00:38:02.376Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone
file = sled-storage/src/dataset.rs:303
|
| 335 | 2024-12-19T00:38:02.548Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b
file = sled-storage/src/dataset.rs:389
|
| 336 | 2024-12-19T00:38:02.556Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:440
|
| 337 | 2024-12-19T00:38:02.556Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
file = sled-storage/src/resources.rs:357
|
| 338 | 2024-12-19T00:38:02.556Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:342
|
| 339 | 2024-12-19T00:38:02.557Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 340 | 2024-12-19T00:38:02.557Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
path = /scratch/u2_4.vdev
|
| 341 | 2024-12-19T00:38:02.566Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev
file = sled-hardware/src/disk.rs:367
|
| 342 | 2024-12-19T00:38:02.566Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool)
|
| 343 | 2024-12-19T00:38:02.662Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/dataset.rs:196
zpool = ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }
|
| 344 | 2024-12-19T00:38:02.670Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/dataset.rs:240
|
| 345 | 2024-12-19T00:38:02.670Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:242
|
| 346 | 2024-12-19T00:38:02.670Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:246
|
| 347 | 2024-12-19T00:38:02.670Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
epoch = 0
file = sled-storage/src/dataset.rs:248
|
| 348 | 2024-12-19T00:38:02.670Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 349 | 2024-12-19T00:38:02.670Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 350 | 2024-12-19T00:38:02.670Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:260
|
| 351 | 2024-12-19T00:38:02.706Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 352 | 2024-12-19T00:38:02.714Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone
file = sled-storage/src/dataset.rs:303
|
| 353 | 2024-12-19T00:38:02.885Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f
file = sled-storage/src/dataset.rs:389
|
| 354 | 2024-12-19T00:38:02.892Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:440
|
| 355 | 2024-12-19T00:38:02.892Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
file = sled-storage/src/resources.rs:357
|
| 356 | 2024-12-19T00:38:02.892Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:342
|
| 357 | 2024-12-19T00:38:02.892Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 358 | 2024-12-19T00:38:02.892Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
path = /scratch/u2_5.vdev
|
| 359 | 2024-12-19T00:38:02.902Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev
file = sled-hardware/src/disk.rs:367
|
| 360 | 2024-12-19T00:38:02.903Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = b0ef393a-1bdf-419f-872d-787b91138b01 (zpool)
|
| 361 | 2024-12-19T00:38:02.999Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/dataset.rs:196
zpool = ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }
|
| 362 | 2024-12-19T00:38:03.006Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/dataset.rs:240
|
| 363 | 2024-12-19T00:38:03.006Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:242
|
| 364 | 2024-12-19T00:38:03.006Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:246
|
| 365 | 2024-12-19T00:38:03.006Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
epoch = 0
file = sled-storage/src/dataset.rs:248
|
| 366 | 2024-12-19T00:38:03.006Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 367 | 2024-12-19T00:38:03.007Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 368 | 2024-12-19T00:38:03.007Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:260
|
| 369 | 2024-12-19T00:38:03.040Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 370 | 2024-12-19T00:38:03.048Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone
file = sled-storage/src/dataset.rs:303
|
| 371 | 2024-12-19T00:38:03.059Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)
file = sled-agent/src/probe_manager.rs:231
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 372 | 2024-12-19T00:38:03.224Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_b0ef393a-1bdf-419f-872d-787b91138b01
file = sled-storage/src/dataset.rs:389
|
| 373 | 2024-12-19T00:38:03.232Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:440
|
| 374 | 2024-12-19T00:38:03.232Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
file = sled-storage/src/resources.rs:357
|
| 375 | 2024-12-19T00:38:03.232Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:342
|
| 376 | 2024-12-19T00:38:03.232Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 377 | 2024-12-19T00:38:03.232Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
path = /scratch/u2_6.vdev
|
| 378 | 2024-12-19T00:38:03.242Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev
file = sled-hardware/src/disk.rs:367
|
| 379 | 2024-12-19T00:38:03.242Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool)
|
| 380 | 2024-12-19T00:38:03.335Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/dataset.rs:196
zpool = ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }
|
| 381 | 2024-12-19T00:38:03.343Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/dataset.rs:240
|
| 382 | 2024-12-19T00:38:03.343Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:242
|
| 383 | 2024-12-19T00:38:03.343Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:246
|
| 384 | 2024-12-19T00:38:03.343Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
epoch = 0
file = sled-storage/src/dataset.rs:248
|
| 385 | 2024-12-19T00:38:03.343Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 386 | 2024-12-19T00:38:03.343Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 387 | 2024-12-19T00:38:03.343Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:260
|
| 388 | 2024-12-19T00:38:03.379Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 389 | 2024-12-19T00:38:03.387Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/zone
file = sled-storage/src/dataset.rs:303
|
| 390 | 2024-12-19T00:38:03.562Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0
file = sled-storage/src/dataset.rs:389
|
| 391 | 2024-12-19T00:38:03.570Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:440
|
| 392 | 2024-12-19T00:38:03.570Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
file = sled-storage/src/resources.rs:357
|
| 393 | 2024-12-19T00:38:03.570Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:342
|
| 394 | 2024-12-19T00:38:03.570Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 395 | 2024-12-19T00:38:03.570Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
path = /scratch/u2_7.vdev
|
| 396 | 2024-12-19T00:38:03.582Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev
file = sled-hardware/src/disk.rs:367
|
| 397 | 2024-12-19T00:38:03.582Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool)
|
| 398 | 2024-12-19T00:38:03.681Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/dataset.rs:196
zpool = ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }
|
| 399 | 2024-12-19T00:38:03.689Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/dataset.rs:240
|
| 400 | 2024-12-19T00:38:03.689Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:242
|
| 401 | 2024-12-19T00:38:03.689Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:246
|
| 402 | 2024-12-19T00:38:03.689Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
epoch = 0
file = sled-storage/src/dataset.rs:248
|
| 403 | 2024-12-19T00:38:03.689Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 404 | 2024-12-19T00:38:03.689Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 405 | 2024-12-19T00:38:03.689Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:260
|
| 406 | 2024-12-19T00:38:03.693Z | WARN | SledAgent (producer-registration-task): failed to lookup Nexus IP, will retry
delay = 13.775685162s
error = "proto error: io error: No route to host (os error 148)"
file = oximeter/producer/src/lib.rs:387
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 407 | 2024-12-19T00:38:03.725Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 408 | 2024-12-19T00:38:03.733Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone
file = sled-storage/src/dataset.rs:303
|
| 409 | 2024-12-19T00:38:03.909Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_94913555-b694-4ecc-9c16-7af8c3b569ae
file = sled-storage/src/dataset.rs:389
|
| 410 | 2024-12-19T00:38:03.917Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:440
|
| 411 | 2024-12-19T00:38:03.917Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
file = sled-storage/src/resources.rs:357
|
| 412 | 2024-12-19T00:38:03.917Z | INFO | SledAgent (StorageResources): Managing disk
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:342
|
| 413 | 2024-12-19T00:38:03.917Z | INFO | SledAgent (StorageResources): Invoking Disk::new on an unmanaged disk
file = sled-storage/src/resources.rs:419
|
| 414 | 2024-12-19T00:38:03.917Z | INFO | SledAgent (StorageResources): Invoking SyntheticDisk::new
file = sled-storage/src/disk.rs:65
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
path = /scratch/u2_8.vdev
|
| 415 | 2024-12-19T00:38:03.928Z | INFO | SledAgent (StorageResources): GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev
file = sled-hardware/src/disk.rs:367
|
| 416 | 2024-12-19T00:38:03.928Z | INFO | SledAgent (StorageResources): Formatting zpool with requested ID
file = sled-hardware/src/disk.rs:373
id = d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool)
|
| 417 | 2024-12-19T00:38:04.027Z | INFO | SledAgent (StorageResources): Ensuring zpool has datasets
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/dataset.rs:196
zpool = ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }
|
| 418 | 2024-12-19T00:38:04.036Z | INFO | SledAgent (StorageResources): Loading latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/dataset.rs:240
|
| 419 | 2024-12-19T00:38:04.036Z | INFO | SledAgent (StorageResources): Loaded latest secret
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:242
|
| 420 | 2024-12-19T00:38:04.036Z | INFO | SledAgent (StorageResources): Retrieving key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:246
|
| 421 | 2024-12-19T00:38:04.036Z | INFO | SledAgent (StorageResources): Got key
disk_id = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
epoch = 0
file = sled-storage/src/dataset.rs:248
|
| 422 | 2024-12-19T00:38:04.036Z | INFO | SledAgent (StorageResources): About to create keyfile
file = sled-storage/src/keyfile.rs:30
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 423 | 2024-12-19T00:38:04.036Z | INFO | SledAgent (StorageResources): Created keyfile
file = sled-storage/src/keyfile.rs:34
path = Keypath("/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key")
|
| 424 | 2024-12-19T00:38:04.036Z | INFO | SledAgent (StorageResources): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:260
|
| 425 | 2024-12-19T00:38:04.061Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)
file = sled-agent/src/probe_manager.rs:231
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 426 | 2024-12-19T00:38:04.073Z | INFO | SledAgent (StorageResources): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:53
|
| 427 | 2024-12-19T00:38:04.081Z | INFO | SledAgent (StorageResources): Automatically destroying dataset: oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone
file = sled-storage/src/dataset.rs:303
|
| 428 | 2024-12-19T00:38:04.255Z | INFO | SledAgent (StorageResources): Looking for unencrypted datasets in oxp_d634c5ac-cef9-463f-93e8-f083982a2a97
file = sled-storage/src/dataset.rs:389
|
| 429 | 2024-12-19T00:38:04.263Z | INFO | SledAgent (StorageResources): Disk::new completed successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:440
|
| 430 | 2024-12-19T00:38:04.263Z | INFO | SledAgent (StorageResources): Disk management started successfully
disk_identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
file = sled-storage/src/resources.rs:357
|
| 431 | 2024-12-19T00:38:04.263Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/.omicron-physical-disks.json.tmp
file = common/src/ledger.rs:199
request = omicron_physical_disks_ensure
|
| 432 | 2024-12-19T00:38:04.263Z | INFO | SledAgent (StorageMonitor): Received storage manager update
disks = AllDisks { generation: Generation(2), inner: AllDisksInner { values: {DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/m2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_0.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_1.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_2.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_3.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_4.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_5.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_6.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_7.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External } })), DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: "/scratch/u2_8.vdev", identity: DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some("SYNTH1")] } }, zpool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External } }))} }, mount_config: MountConfig { root: "/", synthetic_disk_root: "/var/tmp" } }
file = sled-agent/src/storage_monitor.rs:93
|
| 433 | 2024-12-19T00:38:04.263Z | INFO | SledAgent (DumpSetup-worker): Updated view of disks
core_datasets = 0
debug_datasets = 0
dump_slices = 0
file = sled-agent/src/dump_setup.rs:580
|
| 434 | 2024-12-19T00:38:04.263Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 435 | 2024-12-19T00:38:04.264Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/.omicron-physical-disks.json.tmp
file = common/src/ledger.rs:199
request = omicron_physical_disks_ensure
|
| 436 | 2024-12-19T00:38:04.264Z | INFO | SledAgent: physical disks ensure: Updated storage
file = sled-agent/src/sled_agent.rs:900
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 437 | 2024-12-19T00:38:04.264Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 438 | 2024-12-19T00:38:04.264Z | INFO | SledAgent: physical disks ensure: Propagating new generation of disks
file = sled-agent/src/sled_agent.rs:926
generation = Generation(2)
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 439 | 2024-12-19T00:38:04.264Z | INFO | SledAgent: physical disks ensure: Updated storage monitor
file = sled-agent/src/sled_agent.rs:931
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 440 | 2024-12-19T00:38:04.264Z | INFO | SledAgent: physical disks ensure: Updated zone bundler
file = sled-agent/src/sled_agent.rs:936
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 441 | 2024-12-19T00:38:04.264Z | INFO | SledAgent (ProbeManager): use_only_these_disks: Processing new request
file = sled-agent/src/probe_manager.rs:122
gen = Generation(2)
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 442 | 2024-12-19T00:38:04.264Z | INFO | SledAgent: physical disks ensure: Updated probes
file = sled-agent/src/sled_agent.rs:942
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 443 | 2024-12-19T00:38:04.264Z | INFO | SledAgent (InstanceManager): use_only_these_disks: Processing new request
file = sled-agent/src/instance_manager.rs:799
gen = Generation(2)
|
| 444 | 2024-12-19T00:38:04.264Z | INFO | SledAgent: physical disks ensure: Updated instances
file = sled-agent/src/sled_agent.rs:947
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 445 | 2024-12-19T00:38:04.264Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 3803010
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:45008
req_id = 15bb1bdd-5791-4ce2-9733-f93df7355451
response_code = 200
uri = /omicron-physical-disks
|
| 446 | 2024-12-19T00:38:04.296Z | INFO | SledAgent (RSS): attempting to set up sled's datasets: DatasetsConfig { generation: Generation(1), datasets: {02bdb46f-1f7a-4495-8e8a-a1c8f0a03bc0 (dataset): DatasetConfig { id: 02bdb46f-1f7a-4495-8e8a-a1c8f0a03bc0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 08299bfb-0800-4ca8-9b8d-9b576960e166 (dataset): DatasetConfig { id: 08299bfb-0800-4ca8-9b8d-9b576960e166 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 09cc2927-2cf5-4413-8be0-f1ddbc79984d (dataset): DatasetConfig { id: 09cc2927-2cf5-4413-8be0-f1ddbc79984d (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 10dda82e-9265-49e8-b8f3-2a4a9c4c164b (dataset): DatasetConfig { id: 10dda82e-9265-49e8-b8f3-2a4a9c4c164b (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_7acb7f99-08f6-422b-b4ef-5456a5a44c23" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 125f36d7-301e-406a-bc11-db67930c216a (dataset): DatasetConfig { id: 125f36d7-301e-406a-bc11-db67930c216a (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 1356fa90-9862-4602-8bec-987f6eefba47 (dataset): DatasetConfig { id: 1356fa90-9862-4602-8bec-987f6eefba47 (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1542d45f-d5fe-49f7-aac2-c5dc5b736508 (dataset): DatasetConfig { id: 1542d45f-d5fe-49f7-aac2-c5dc5b736508 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: "oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1ba3c1dd-2484-4777-8624-adf9adabad51 (dataset): DatasetConfig { id: 1ba3c1dd-2484-4777-8624-adf9adabad51 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_pantry_70eb21d3-150c-4d21-bcc3-bbf3bd06c293" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1bd3a984-3b13-48a9-9fd9-5a85e418f46b (dataset): DatasetConfig { id: 1bd3a984-3b13-48a9-9fd9-5a85e418f46b (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2385dbc1-3195-45e0-8662-43c4a515014c (dataset): DatasetConfig { id: 2385dbc1-3195-45e0-8662-43c4a515014c (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 25705b62-3c7a-45bd-9cba-99bd5767da59 (dataset): DatasetConfig { id: 25705b62-3c7a-45bd-9cba-99bd5767da59 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_d5087ea8-b098-443c-b9ed-af8abfbf27e8" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 26790d95-ebdd-44fd-86e5-9dd4ba3e4f21 (dataset): DatasetConfig { id: 26790d95-ebdd-44fd-86e5-9dd4ba3e4f21 (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 26e0b6b0-ce70-43ca-980b-0cf12a292bcf (dataset): DatasetConfig { id: 26e0b6b0-ce70-43ca-980b-0cf12a292bcf (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 2d1e47ac-ebac-444a-b49e-9d3573199828 (dataset): DatasetConfig { id: 2d1e47ac-ebac-444a-b49e-9d3573199828 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 31e811be-26c1-4ef9-b32e-05600feef342 (dataset): DatasetConfig { id: 31e811be-26c1-4ef9-b32e-05600feef342 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_33c09be7-40a9-46e5-829a-6af4b245eb8d" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 338b98ec-77e1-415d-8738-e1eea139be02 (dataset): DatasetConfig { id: 338b98ec-77e1-415d-8738-e1eea139be02 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_6bb5054e-2e33-47c5-8591-838209d609a0" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3890ce90-eb11-4f7f-b73f-41bc8c3549cd (dataset): DatasetConfig { id: 3890ce90-eb11-4f7f-b73f-41bc8c3549cd (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 3a270f45-f343-4a4c-a795-e1d1d4026117 (dataset): DatasetConfig { id: 3a270f45-f343-4a4c-a795-e1d1d4026117 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a605806-3dd8-48f6-a271-4921a40cab01 (dataset): DatasetConfig { id: 3a605806-3dd8-48f6-a271-4921a40cab01 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4801adcb-51da-4c43-a6b8-74c9ab7d469e (dataset): DatasetConfig { id: 4801adcb-51da-4c43-a6b8-74c9ab7d469e (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: "oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 53e73ca6-22a3-4760-8d8c-13ba65fd0138 (dataset): DatasetConfig { id: 53e73ca6-22a3-4760-8d8c-13ba65fd0138 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 57924c57-7a89-42cf-b574-6af7898fbf20 (dataset): DatasetConfig { id: 57924c57-7a89-42cf-b574-6af7898fbf20 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 57d1f209-4bbf-4b28-8fc6-71af4aa89766 (dataset): DatasetConfig { id: 57d1f209-4bbf-4b28-8fc6-71af4aa89766 (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 57d5be5f-b1fb-493e-abd9-49787f94f8bd (dataset): DatasetConfig { id: 57d5be5f-b1fb-493e-abd9-49787f94f8bd (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: TransientZone { name: "oxz_nexus_6a23c2a5-006e-4c29-a54c-390deecea882" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 58434d8e-917a-40ea-a69c-83af306968fe (dataset): DatasetConfig { id: 58434d8e-917a-40ea-a69c-83af306968fe (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_pantry_60daa419-d06f-44da-8437-507d9480bdc5" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5b27c472-485e-4e9d-9aba-7cd17c6bd10d (dataset): DatasetConfig { id: 5b27c472-485e-4e9d-9aba-7cd17c6bd10d (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: "oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5e7c3ef9-e5f3-4f5b-afd5-97a14d72a56c (dataset): DatasetConfig { id: 5e7c3ef9-e5f3-4f5b-afd5-97a14d72a56c (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 626f8ac1-7e2f-4555-a521-96f6d9d1c49f (dataset): DatasetConfig { id: 626f8ac1-7e2f-4555-a521-96f6d9d1c49f (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 628dbe7b-bcea-4ffc-9f4e-c56ad9549e5e (dataset): DatasetConfig { id: 628dbe7b-bcea-4ffc-9f4e-c56ad9549e5e (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 63082ad4-1646-44cb-a619-f99202ef3e1d (dataset): DatasetConfig { id: 63082ad4-1646-44cb-a619-f99202ef3e1d (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: "oxz_oximeter_2e431855-2d60-4eca-bbca-519c0a7d8124" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6cf98f37-21a1-4ff0-b9a6-3d0efc3ee8be (dataset): DatasetConfig { id: 6cf98f37-21a1-4ff0-b9a6-3d0efc3ee8be (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_1bfa1410-b334-4a2b-ac4e-77fa9c04ad35" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6ee13928-1186-4e68-ae12-f5f0880b9bda (dataset): DatasetConfig { id: 6ee13928-1186-4e68-ae12-f5f0880b9bda (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 749db2ac-f07c-459b-8dfe-2e14e25a342c (dataset): DatasetConfig { id: 749db2ac-f07c-459b-8dfe-2e14e25a342c (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 758f88cf-7734-4193-a836-12f01f4bbff8 (dataset): DatasetConfig { id: 758f88cf-7734-4193-a836-12f01f4bbff8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_pantry_51618710-2d9e-4911-bfc3-577586e96ca3" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 787d82fb-5c78-42bb-8d95-7bd9d94d3c37 (dataset): DatasetConfig { id: 787d82fb-5c78-42bb-8d95-7bd9d94d3c37 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: "oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7fd0b8db-fc34-4652-b220-93eaa7beee17 (dataset): DatasetConfig { id: 7fd0b8db-fc34-4652-b220-93eaa7beee17 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8948f738-5705-4437-8259-8bb3c045db11 (dataset): DatasetConfig { id: 8948f738-5705-4437-8259-8bb3c045db11 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8e32d5e5-5fbe-40f4-87a8-39a7939aa467 (dataset): DatasetConfig { id: 8e32d5e5-5fbe-40f4-87a8-39a7939aa467 (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9237f071-5643-491d-9e34-1f622d5fe205 (dataset): DatasetConfig { id: 9237f071-5643-491d-9e34-1f622d5fe205 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 93bcddc6-63cc-4c1b-a60f-47b0fb416c23 (dataset): DatasetConfig { id: 93bcddc6-63cc-4c1b-a60f-47b0fb416c23 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 99c77c44-c34f-4aa4-9f16-3e4edb45d1ec (dataset): DatasetConfig { id: 99c77c44-c34f-4aa4-9f16-3e4edb45d1ec (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9aa0d225-1893-4fc0-8d5d-40cf890c4067 (dataset): DatasetConfig { id: 9aa0d225-1893-4fc0-8d5d-40cf890c4067 (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9e64b929-13c2-4e94-ba4f-96f15d4ba89a (dataset): DatasetConfig { id: 9e64b929-13c2-4e94-ba4f-96f15d4ba89a (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a0d4c349-9c98-4a73-a925-7147d15826af (dataset): DatasetConfig { id: a0d4c349-9c98-4a73-a925-7147d15826af (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a6d1c34e-ab1b-4003-ad9c-83c0ad244a7d (dataset): DatasetConfig { id: a6d1c34e-ab1b-4003-ad9c-83c0ad244a7d (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a8df2468-e878-45e6-832a-fddd2225182c (dataset): DatasetConfig { id: a8df2468-e878-45e6-832a-fddd2225182c (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: "oxz_nexus_72e950cb-aef5-40dd-aa92-293783dc94a9" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a984653e-3079-4570-a013-ae3c15e5ce07 (dataset): DatasetConfig { id: a984653e-3079-4570-a013-ae3c15e5ce07 (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_2306f05d-d638-4515-af92-dc4836424084" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, aca6795e-4c1e-43ed-913e-0306275b8e4b (dataset): DatasetConfig { id: aca6795e-4c1e-43ed-913e-0306275b8e4b (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_1dfeb85c-baf9-49e2-9020-eaca2cf6ce8c" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b2a6fb9d-7c72-4a2a-8ea1-d8423ce57969 (dataset): DatasetConfig { id: b2a6fb9d-7c72-4a2a-8ea1-d8423ce57969 (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_9785445e-9baa-4024-8a89-6b305fb3e36a" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b4c03dc8-848c-42c0-852c-45d1e019fdfe (dataset): DatasetConfig { id: b4c03dc8-848c-42c0-852c-45d1e019fdfe (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b667d6bf-cd95-46fe-a358-22481d6d053f (dataset): DatasetConfig { id: b667d6bf-cd95-46fe-a358-22481d6d053f (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cc2afb42-705b-4632-bd97-aa8aaec066a4 (dataset): DatasetConfig { id: cc2afb42-705b-4632-bd97-aa8aaec066a4 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cf2cf5d1-07e2-42cf-8928-1b07f767f047 (dataset): DatasetConfig { id: cf2cf5d1-07e2-42cf-8928-1b07f767f047 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, d2b8c34d-dc1a-46e0-96ec-d6982badb0e6 (dataset): DatasetConfig { id: d2b8c34d-dc1a-46e0-96ec-d6982badb0e6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: "oxz_external_dns_7b1c75dd-240b-4017-a160-edd46f1b3d00" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d73cd6fb-47af-40a8-a0fa-94b371a46ad3 (dataset): DatasetConfig { id: d73cd6fb-47af-40a8-a0fa-94b371a46ad3 (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: TransientZone { name: "oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d9e5b916-9c1b-4b59-b319-26e322fbdb61 (dataset): DatasetConfig { id: d9e5b916-9c1b-4b59-b319-26e322fbdb61 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: "oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dbda3008-9dab-4080-ad6f-45c5fb06ba02 (dataset): DatasetConfig { id: dbda3008-9dab-4080-ad6f-45c5fb06ba02 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, dc0beae4-4e1b-420d-8bde-db32c20e0766 (dataset): DatasetConfig { id: dc0beae4-4e1b-420d-8bde-db32c20e0766 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: "oxz_nexus_445c7eec-d66d-460b-9a24-27198fcf6179" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e0ba913d-d324-479a-9c68-11b2e74233ff (dataset): DatasetConfig { id: e0ba913d-d324-479a-9c68-11b2e74233ff (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: TransientZone { name: "oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e3405a87-6129-4340-b145-44b361ae589f (dataset): DatasetConfig { id: e3405a87-6129-4340-b145-44b361ae589f (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZone { name: "oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e76a48c5-4ab0-45e0-8a91-7b4c18c5c3b0 (dataset): DatasetConfig { id: e76a48c5-4ab0-45e0-8a91-7b4c18c5c3b0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f18c0c55-424d-4a31-a713-40eb48be250e (dataset): DatasetConfig { id: f18c0c55-424d-4a31-a713-40eb48be250e (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZone { name: "oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f42f9875-f490-4a91-ac26-ac14081052dd (dataset): DatasetConfig { id: f42f9875-f490-4a91-ac26-ac14081052dd (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f4a0219f-54a0-4ca0-b405-241fecf8f740 (dataset): DatasetConfig { id: f4a0219f-54a0-4ca0-b405-241fecf8f740 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: "oxz_clickhouse_20d7411f-1498-441e-89c3-f4071a3d9fb5" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f988315b-a9ab-4cd6-8440-4fb757c7615b (dataset): DatasetConfig { id: f988315b-a9ab-4cd6-8440-4fb757c7615b (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_1cc9eeb4-0d42-4938-8c8b-a21df5e97007" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fc2fb1b2-6d4b-40c4-ab61-aaa08209b143 (dataset): DatasetConfig { id: fc2fb1b2-6d4b-40c4-ab61-aaa08209b143 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: "oxz_external_dns_77622b51-9ad8-477c-8380-8cae282506aa" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }
file = sled-agent/src/rack_setup/service.rs:460
sled_address = [fd00:1122:3344:101::1]:12345
|
| 447 | 2024-12-19T00:38:04.296Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:46149
|
| 448 | 2024-12-19T00:38:04.296Z | INFO | SledAgent: datasets ensure
file = sled-agent/src/sled_agent.rs:862
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 449 | 2024-12-19T00:38:04.296Z | INFO | SledAgent (StorageManager): Received DatasetsEnsure { config: DatasetsConfig { generation: Generation(1), datasets: {02bdb46f-1f7a-4495-8e8a-a1c8f0a03bc0 (dataset): DatasetConfig { id: 02bdb46f-1f7a-4495-8e8a-a1c8f0a03bc0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 08299bfb-0800-4ca8-9b8d-9b576960e166 (dataset): DatasetConfig { id: 08299bfb-0800-4ca8-9b8d-9b576960e166 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 09cc2927-2cf5-4413-8be0-f1ddbc79984d (dataset): DatasetConfig { id: 09cc2927-2cf5-4413-8be0-f1ddbc79984d (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 10dda82e-9265-49e8-b8f3-2a4a9c4c164b (dataset): DatasetConfig { id: 10dda82e-9265-49e8-b8f3-2a4a9c4c164b (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_7acb7f99-08f6-422b-b4ef-5456a5a44c23" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 125f36d7-301e-406a-bc11-db67930c216a (dataset): DatasetConfig { id: 125f36d7-301e-406a-bc11-db67930c216a (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 1356fa90-9862-4602-8bec-987f6eefba47 (dataset): DatasetConfig { id: 1356fa90-9862-4602-8bec-987f6eefba47 (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1542d45f-d5fe-49f7-aac2-c5dc5b736508 (dataset): DatasetConfig { id: 1542d45f-d5fe-49f7-aac2-c5dc5b736508 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: "oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1ba3c1dd-2484-4777-8624-adf9adabad51 (dataset): DatasetConfig { id: 1ba3c1dd-2484-4777-8624-adf9adabad51 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_pantry_70eb21d3-150c-4d21-bcc3-bbf3bd06c293" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1bd3a984-3b13-48a9-9fd9-5a85e418f46b (dataset): DatasetConfig { id: 1bd3a984-3b13-48a9-9fd9-5a85e418f46b (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2385dbc1-3195-45e0-8662-43c4a515014c (dataset): DatasetConfig { id: 2385dbc1-3195-45e0-8662-43c4a515014c (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 25705b62-3c7a-45bd-9cba-99bd5767da59 (dataset): DatasetConfig { id: 25705b62-3c7a-45bd-9cba-99bd5767da59 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_d5087ea8-b098-443c-b9ed-af8abfbf27e8" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 26790d95-ebdd-44fd-86e5-9dd4ba3e4f21 (dataset): DatasetConfig { id: 26790d95-ebdd-44fd-86e5-9dd4ba3e4f21 (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 26e0b6b0-ce70-43ca-980b-0cf12a292bcf (dataset): DatasetConfig { id: 26e0b6b0-ce70-43ca-980b-0cf12a292bcf (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 2d1e47ac-ebac-444a-b49e-9d3573199828 (dataset): DatasetConfig { id: 2d1e47ac-ebac-444a-b49e-9d3573199828 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 31e811be-26c1-4ef9-b32e-05600feef342 (dataset): DatasetConfig { id: 31e811be-26c1-4ef9-b32e-05600feef342 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_33c09be7-40a9-46e5-829a-6af4b245eb8d" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 338b98ec-77e1-415d-8738-e1eea139be02 (dataset): DatasetConfig { id: 338b98ec-77e1-415d-8738-e1eea139be02 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_6bb5054e-2e33-47c5-8591-838209d609a0" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3890ce90-eb11-4f7f-b73f-41bc8c3549cd (dataset): DatasetConfig { id: 3890ce90-eb11-4f7f-b73f-41bc8c3549cd (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 3a270f45-f343-4a4c-a795-e1d1d4026117 (dataset): DatasetConfig { id: 3a270f45-f343-4a4c-a795-e1d1d4026117 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a605806-3dd8-48f6-a271-4921a40cab01 (dataset): DatasetConfig { id: 3a605806-3dd8-48f6-a271-4921a40cab01 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4801adcb-51da-4c43-a6b8-74c9ab7d469e (dataset): DatasetConfig { id: 4801adcb-51da-4c43-a6b8-74c9ab7d469e (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: "oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 53e73ca6-22a3-4760-8d8c-13ba65fd0138 (dataset): DatasetConfig { id: 53e73ca6-22a3-4760-8d8c-13ba65fd0138 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 57924c57-7a89-42cf-b574-6af7898fbf20 (dataset): DatasetConfig { id: 57924c57-7a89-42cf-b574-6af7898fbf20 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 57d1f209-4bbf-4b28-8fc6-71af4aa89766 (dataset): DatasetConfig { id: 57d1f209-4bbf-4b28-8fc6-71af4aa89766 (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 57d5be5f-b1fb-493e-abd9-49787f94f8bd (dataset): DatasetConfig { id: 57d5be5f-b1fb-493e-abd9-49787f94f8bd (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: TransientZone { name: "oxz_nexus_6a23c2a5-006e-4c29-a54c-390deecea882" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 58434d8e-917a-40ea-a69c-83af306968fe (dataset): DatasetConfig { id: 58434d8e-917a-40ea-a69c-83af306968fe (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_pantry_60daa419-d06f-44da-8437-507d9480bdc5" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5b27c472-485e-4e9d-9aba-7cd17c6bd10d (dataset): DatasetConfig { id: 5b27c472-485e-4e9d-9aba-7cd17c6bd10d (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: "oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5e7c3ef9-e5f3-4f5b-afd5-97a14d72a56c (dataset): DatasetConfig { id: 5e7c3ef9-e5f3-4f5b-afd5-97a14d72a56c (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 626f8ac1-7e2f-4555-a521-96f6d9d1c49f (dataset): DatasetConfig { id: 626f8ac1-7e2f-4555-a521-96f6d9d1c49f (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 628dbe7b-bcea-4ffc-9f4e-c56ad9549e5e (dataset): DatasetConfig { id: 628dbe7b-bcea-4ffc-9f4e-c56ad9549e5e (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 63082ad4-1646-44cb-a619-f99202ef3e1d (dataset): DatasetConfig { id: 63082ad4-1646-44cb-a619-f99202ef3e1d (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: "oxz_oximeter_2e431855-2d60-4eca-bbca-519c0a7d8124" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6cf98f37-21a1-4ff0-b9a6-3d0efc3ee8be (dataset): DatasetConfig { id: 6cf98f37-21a1-4ff0-b9a6-3d0efc3ee8be (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_1bfa1410-b334-4a2b-ac4e-77fa9c04ad35" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6ee13928-1186-4e68-ae12-f5f0880b9bda (dataset): DatasetConfig { id: 6ee13928-1186-4e68-ae12-f5f0880b9bda (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 749db2ac-f07c-459b-8dfe-2e14e25a342c (dataset): DatasetConfig { id: 749db2ac-f07c-459b-8dfe-2e14e25a342c (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 758f88cf-7734-4193-a836-12f01f4bbff8 (dataset): DatasetConfig { id: 758f88cf-7734-4193-a836-12f01f4bbff8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_pantry_51618710-2d9e-4911-bfc3-577586e96ca3" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 787d82fb-5c78-42bb-8d95-7bd9d94d3c37 (dataset): DatasetConfig { id: 787d82fb-5c78-42bb-8d95-7bd9d94d3c37 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: "oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7fd0b8db-fc34-4652-b220-93eaa7beee17 (dataset): DatasetConfig { id: 7fd0b8db-fc34-4652-b220-93eaa7beee17 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8948f738-5705-4437-8259-8bb3c045db11 (dataset): DatasetConfig { id: 8948f738-5705-4437-8259-8bb3c045db11 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8e32d5e5-5fbe-40f4-87a8-39a7939aa467 (dataset): DatasetConfig { id: 8e32d5e5-5fbe-40f4-87a8-39a7939aa467 (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9237f071-5643-491d-9e34-1f622d5fe205 (dataset): DatasetConfig { id: 9237f071-5643-491d-9e34-1f622d5fe205 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 93bcddc6-63cc-4c1b-a60f-47b0fb416c23 (dataset): DatasetConfig { id: 93bcddc6-63cc-4c1b-a60f-47b0fb416c23 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 99c77c44-c34f-4aa4-9f16-3e4edb45d1ec (dataset): DatasetConfig { id: 99c77c44-c34f-4aa4-9f16-3e4edb45d1ec (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9aa0d225-1893-4fc0-8d5d-40cf890c4067 (dataset): DatasetConfig { id: 9aa0d225-1893-4fc0-8d5d-40cf890c4067 (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9e64b929-13c2-4e94-ba4f-96f15d4ba89a (dataset): DatasetConfig { id: 9e64b929-13c2-4e94-ba4f-96f15d4ba89a (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a0d4c349-9c98-4a73-a925-7147d15826af (dataset): DatasetConfig { id: a0d4c349-9c98-4a73-a925-7147d15826af (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a6d1c34e-ab1b-4003-ad9c-83c0ad244a7d (dataset): DatasetConfig { id: a6d1c34e-ab1b-4003-ad9c-83c0ad244a7d (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a8df2468-e878-45e6-832a-fddd2225182c (dataset): DatasetConfig { id: a8df2468-e878-45e6-832a-fddd2225182c (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: "oxz_nexus_72e950cb-aef5-40dd-aa92-293783dc94a9" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a984653e-3079-4570-a013-ae3c15e5ce07 (dataset): DatasetConfig { id: a984653e-3079-4570-a013-ae3c15e5ce07 (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_2306f05d-d638-4515-af92-dc4836424084" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, aca6795e-4c1e-43ed-913e-0306275b8e4b (dataset): DatasetConfig { id: aca6795e-4c1e-43ed-913e-0306275b8e4b (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_1dfeb85c-baf9-49e2-9020-eaca2cf6ce8c" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b2a6fb9d-7c72-4a2a-8ea1-d8423ce57969 (dataset): DatasetConfig { id: b2a6fb9d-7c72-4a2a-8ea1-d8423ce57969 (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_9785445e-9baa-4024-8a89-6b305fb3e36a" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b4c03dc8-848c-42c0-852c-45d1e019fdfe (dataset): DatasetConfig { id: b4c03dc8-848c-42c0-852c-45d1e019fdfe (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b667d6bf-cd95-46fe-a358-22481d6d053f (dataset): DatasetConfig { id: b667d6bf-cd95-46fe-a358-22481d6d053f (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cc2afb42-705b-4632-bd97-aa8aaec066a4 (dataset): DatasetConfig { id: cc2afb42-705b-4632-bd97-aa8aaec066a4 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cf2cf5d1-07e2-42cf-8928-1b07f767f047 (dataset): DatasetConfig { id: cf2cf5d1-07e2-42cf-8928-1b07f767f047 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, d2b8c34d-dc1a-46e0-96ec-d6982badb0e6 (dataset): DatasetConfig { id: d2b8c34d-dc1a-46e0-96ec-d6982badb0e6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: "oxz_external_dns_7b1c75dd-240b-4017-a160-edd46f1b3d00" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d73cd6fb-47af-40a8-a0fa-94b371a46ad3 (dataset): DatasetConfig { id: d73cd6fb-47af-40a8-a0fa-94b371a46ad3 (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: TransientZone { name: "oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d9e5b916-9c1b-4b59-b319-26e322fbdb61 (dataset): DatasetConfig { id: d9e5b916-9c1b-4b59-b319-26e322fbdb61 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: "oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dbda3008-9dab-4080-ad6f-45c5fb06ba02 (dataset): DatasetConfig { id: dbda3008-9dab-4080-ad6f-45c5fb06ba02 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, dc0beae4-4e1b-420d-8bde-db32c20e0766 (dataset): DatasetConfig { id: dc0beae4-4e1b-420d-8bde-db32c20e0766 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: "oxz_nexus_445c7eec-d66d-460b-9a24-27198fcf6179" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e0ba913d-d324-479a-9c68-11b2e74233ff (dataset): DatasetConfig { id: e0ba913d-d324-479a-9c68-11b2e74233ff (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: TransientZone { name: "oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e3405a87-6129-4340-b145-44b361ae589f (dataset): DatasetConfig { id: e3405a87-6129-4340-b145-44b361ae589f (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZone { name: "oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e76a48c5-4ab0-45e0-8a91-7b4c18c5c3b0 (dataset): DatasetConfig { id: e76a48c5-4ab0-45e0-8a91-7b4c18c5c3b0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f18c0c55-424d-4a31-a713-40eb48be250e (dataset): DatasetConfig { id: f18c0c55-424d-4a31-a713-40eb48be250e (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZone { name: "oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f42f9875-f490-4a91-ac26-ac14081052dd (dataset): DatasetConfig { id: f42f9875-f490-4a91-ac26-ac14081052dd (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f4a0219f-54a0-4ca0-b405-241fecf8f740 (dataset): DatasetConfig { id: f4a0219f-54a0-4ca0-b405-241fecf8f740 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: "oxz_clickhouse_20d7411f-1498-441e-89c3-f4071a3d9fb5" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f988315b-a9ab-4cd6-8440-4fb757c7615b (dataset): DatasetConfig { id: f988315b-a9ab-4cd6-8440-4fb757c7615b (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: TransientZone { name: "oxz_crucible_1cc9eeb4-0d42-4938-8c8b-a21df5e97007" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fc2fb1b2-6d4b-40c4-ab61-aaa08209b143 (dataset): DatasetConfig { id: fc2fb1b2-6d4b-40c4-ab61-aaa08209b143 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: "oxz_external_dns_77622b51-9ad8-477c-8380-8cae282506aa" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }, tx: ... }
file = sled-storage/src/manager.rs:603
|
| 450 | 2024-12-19T00:38:04.299Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/omicron-datasets.json
file = common/src/ledger.rs:188
request = datasets_ensure
|
| 451 | 2024-12-19T00:38:04.299Z | INFO | SledAgent (StorageManager): No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/omicron-datasets.json
file = common/src/ledger.rs:188
request = datasets_ensure
|
| 452 | 2024-12-19T00:38:04.299Z | INFO | SledAgent (StorageManager): No previously-stored 'requested datasets', creating new ledger
file = sled-storage/src/manager.rs:905
request = datasets_ensure
|
| 453 | 2024-12-19T00:38:04.299Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 454 | 2024-12-19T00:38:04.299Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/clickhouse" }
file = sled-storage/src/manager.rs:1331
|
| 455 | 2024-12-19T00:38:04.385Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 456 | 2024-12-19T00:38:04.385Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/cockroachdb" }
file = sled-storage/src/manager.rs:1331
|
| 457 | 2024-12-19T00:38:04.472Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 458 | 2024-12-19T00:38:04.472Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crucible" }
file = sled-storage/src/manager.rs:1331
|
| 459 | 2024-12-19T00:38:04.558Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 460 | 2024-12-19T00:38:04.558Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_crucible_7acb7f99-08f6-422b-b4ef-5456a5a44c23"), full_name: "oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_crucible_7acb7f99-08f6-422b-b4ef-5456a5a44c23" }
file = sled-storage/src/manager.rs:1331
|
| 461 | 2024-12-19T00:38:04.654Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 462 | 2024-12-19T00:38:04.654Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/debug"), full_name: "oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/debug" }
file = sled-storage/src/manager.rs:1331
|
| 463 | 2024-12-19T00:38:04.719Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 464 | 2024-12-19T00:38:04.719Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone"), full_name: "oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone" }
file = sled-storage/src/manager.rs:1331
|
| 465 | 2024-12-19T00:38:04.787Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 466 | 2024-12-19T00:38:04.787Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf"), full_name: "oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf" }
file = sled-storage/src/manager.rs:1331
|
| 467 | 2024-12-19T00:38:04.887Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 468 | 2024-12-19T00:38:04.887Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_crucible_pantry_70eb21d3-150c-4d21-bcc3-bbf3bd06c293"), full_name: "oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_crucible_pantry_70eb21d3-150c-4d21-bcc3-bbf3bd06c293" }
file = sled-storage/src/manager.rs:1331
|
| 469 | 2024-12-19T00:38:04.987Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 470 | 2024-12-19T00:38:04.987Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone"), full_name: "oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone" }
file = sled-storage/src/manager.rs:1331
|
| 471 | 2024-12-19T00:38:05.054Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 472 | 2024-12-19T00:38:05.054Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/external_dns" }
file = sled-storage/src/manager.rs:1331
|
| 473 | 2024-12-19T00:38:05.064Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)
file = sled-agent/src/probe_manager.rs:231
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 474 | 2024-12-19T00:38:05.143Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 475 | 2024-12-19T00:38:05.143Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_crucible_d5087ea8-b098-443c-b9ed-af8abfbf27e8"), full_name: "oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_crucible_d5087ea8-b098-443c-b9ed-af8abfbf27e8" }
file = sled-storage/src/manager.rs:1331
|
| 476 | 2024-12-19T00:38:05.240Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 477 | 2024-12-19T00:38:05.240Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/debug"), full_name: "oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/debug" }
file = sled-storage/src/manager.rs:1331
|
| 478 | 2024-12-19T00:38:05.307Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 479 | 2024-12-19T00:38:05.307Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/debug"), full_name: "oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/debug" }
file = sled-storage/src/manager.rs:1331
|
| 480 | 2024-12-19T00:38:05.374Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 481 | 2024-12-19T00:38:05.374Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/debug"), full_name: "oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/debug" }
file = sled-storage/src/manager.rs:1331
|
| 482 | 2024-12-19T00:38:05.441Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 483 | 2024-12-19T00:38:05.441Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/zone/oxz_crucible_33c09be7-40a9-46e5-829a-6af4b245eb8d"), full_name: "oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/zone/oxz_crucible_33c09be7-40a9-46e5-829a-6af4b245eb8d" }
file = sled-storage/src/manager.rs:1331
|
| 484 | 2024-12-19T00:38:05.539Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 485 | 2024-12-19T00:38:05.539Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_crucible_6bb5054e-2e33-47c5-8591-838209d609a0"), full_name: "oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_crucible_6bb5054e-2e33-47c5-8591-838209d609a0" }
file = sled-storage/src/manager.rs:1331
|
| 486 | 2024-12-19T00:38:05.639Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 487 | 2024-12-19T00:38:05.639Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/debug"), full_name: "oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/debug" }
file = sled-storage/src/manager.rs:1331
|
| 488 | 2024-12-19T00:38:05.708Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 489 | 2024-12-19T00:38:05.708Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone"), full_name: "oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone" }
file = sled-storage/src/manager.rs:1331
|
| 490 | 2024-12-19T00:38:05.776Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 491 | 2024-12-19T00:38:05.776Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/zone"), full_name: "oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/zone" }
file = sled-storage/src/manager.rs:1331
|
| 492 | 2024-12-19T00:38:05.843Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 493 | 2024-12-19T00:38:05.843Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5"), full_name: "oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5" }
file = sled-storage/src/manager.rs:1331
|
| 494 | 2024-12-19T00:38:05.942Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 495 | 2024-12-19T00:38:05.943Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone"), full_name: "oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone" }
file = sled-storage/src/manager.rs:1331
|
| 496 | 2024-12-19T00:38:06.011Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 497 | 2024-12-19T00:38:06.011Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crucible" }
file = sled-storage/src/manager.rs:1331
|
| 498 | 2024-12-19T00:38:06.067Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)
file = sled-agent/src/probe_manager.rs:231
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 499 | 2024-12-19T00:38:06.100Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 500 | 2024-12-19T00:38:06.100Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/debug"), full_name: "oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/debug" }
file = sled-storage/src/manager.rs:1331
|
| 501 | 2024-12-19T00:38:06.169Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 502 | 2024-12-19T00:38:06.169Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone/oxz_nexus_6a23c2a5-006e-4c29-a54c-390deecea882"), full_name: "oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone/oxz_nexus_6a23c2a5-006e-4c29-a54c-390deecea882" }
file = sled-storage/src/manager.rs:1331
|
| 503 | 2024-12-19T00:38:06.269Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 504 | 2024-12-19T00:38:06.269Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone/oxz_crucible_pantry_60daa419-d06f-44da-8437-507d9480bdc5"), full_name: "oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone/oxz_crucible_pantry_60daa419-d06f-44da-8437-507d9480bdc5" }
file = sled-storage/src/manager.rs:1331
|
| 505 | 2024-12-19T00:38:06.369Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 506 | 2024-12-19T00:38:06.369Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c"), full_name: "oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c" }
file = sled-storage/src/manager.rs:1331
|
| 507 | 2024-12-19T00:38:06.468Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 508 | 2024-12-19T00:38:06.468Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone"), full_name: "oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone" }
file = sled-storage/src/manager.rs:1331
|
| 509 | 2024-12-19T00:38:06.536Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 510 | 2024-12-19T00:38:06.536Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crucible" }
file = sled-storage/src/manager.rs:1331
|
| 511 | 2024-12-19T00:38:06.625Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 512 | 2024-12-19T00:38:06.625Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/b0ef393a-1bdf-419f-872d-787b91138b01/crypt/debug"), full_name: "oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crypt/debug" }
file = sled-storage/src/manager.rs:1331
|
| 513 | 2024-12-19T00:38:06.692Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 514 | 2024-12-19T00:38:06.692Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_oximeter_2e431855-2d60-4eca-bbca-519c0a7d8124"), full_name: "oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_oximeter_2e431855-2d60-4eca-bbca-519c0a7d8124" }
file = sled-storage/src/manager.rs:1331
|
| 515 | 2024-12-19T00:38:06.794Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 516 | 2024-12-19T00:38:06.794Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_crucible_1bfa1410-b334-4a2b-ac4e-77fa9c04ad35"), full_name: "oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_crucible_1bfa1410-b334-4a2b-ac4e-77fa9c04ad35" }
file = sled-storage/src/manager.rs:1331
|
| 517 | 2024-12-19T00:38:06.893Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 518 | 2024-12-19T00:38:06.893Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/cockroachdb" }
file = sled-storage/src/manager.rs:1331
|
| 519 | 2024-12-19T00:38:06.984Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 520 | 2024-12-19T00:38:06.984Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/cockroachdb" }
file = sled-storage/src/manager.rs:1331
|
| 521 | 2024-12-19T00:38:07.070Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)
file = sled-agent/src/probe_manager.rs:231
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 522 | 2024-12-19T00:38:07.073Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 523 | 2024-12-19T00:38:07.073Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_crucible_pantry_51618710-2d9e-4911-bfc3-577586e96ca3"), full_name: "oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_crucible_pantry_51618710-2d9e-4911-bfc3-577586e96ca3" }
file = sled-storage/src/manager.rs:1331
|
| 524 | 2024-12-19T00:38:07.172Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 525 | 2024-12-19T00:38:07.172Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29"), full_name: "oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29" }
file = sled-storage/src/manager.rs:1331
|
| 526 | 2024-12-19T00:38:07.271Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 527 | 2024-12-19T00:38:07.271Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/cockroachdb" }
file = sled-storage/src/manager.rs:1331
|
| 528 | 2024-12-19T00:38:07.362Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 529 | 2024-12-19T00:38:07.362Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crucible" }
file = sled-storage/src/manager.rs:1331
|
| 530 | 2024-12-19T00:38:07.448Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 531 | 2024-12-19T00:38:07.448Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone"), full_name: "oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone" }
file = sled-storage/src/manager.rs:1331
|
| 532 | 2024-12-19T00:38:07.515Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 533 | 2024-12-19T00:38:07.516Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crucible" }
file = sled-storage/src/manager.rs:1331
|
| 534 | 2024-12-19T00:38:07.605Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 535 | 2024-12-19T00:38:07.605Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/external_dns" }
file = sled-storage/src/manager.rs:1331
|
| 536 | 2024-12-19T00:38:07.698Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 537 | 2024-12-19T00:38:07.698Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/internal_dns" }
file = sled-storage/src/manager.rs:1331
|
| 538 | 2024-12-19T00:38:07.782Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 539 | 2024-12-19T00:38:07.782Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crucible" }
file = sled-storage/src/manager.rs:1331
|
| 540 | 2024-12-19T00:38:07.866Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 541 | 2024-12-19T00:38:07.866Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crucible" }
file = sled-storage/src/manager.rs:1331
|
| 542 | 2024-12-19T00:38:07.950Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 543 | 2024-12-19T00:38:07.950Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/internal_dns" }
file = sled-storage/src/manager.rs:1331
|
| 544 | 2024-12-19T00:38:08.040Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 545 | 2024-12-19T00:38:08.040Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crucible" }
file = sled-storage/src/manager.rs:1331
|
| 546 | 2024-12-19T00:38:08.072Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)
file = sled-agent/src/probe_manager.rs:231
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 547 | 2024-12-19T00:38:08.126Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 548 | 2024-12-19T00:38:08.126Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_nexus_72e950cb-aef5-40dd-aa92-293783dc94a9"), full_name: "oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_nexus_72e950cb-aef5-40dd-aa92-293783dc94a9" }
file = sled-storage/src/manager.rs:1331
|
| 549 | 2024-12-19T00:38:08.222Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 550 | 2024-12-19T00:38:08.222Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone/oxz_crucible_2306f05d-d638-4515-af92-dc4836424084"), full_name: "oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone/oxz_crucible_2306f05d-d638-4515-af92-dc4836424084" }
file = sled-storage/src/manager.rs:1331
|
| 551 | 2024-12-19T00:38:08.322Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 552 | 2024-12-19T00:38:08.323Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone/oxz_crucible_1dfeb85c-baf9-49e2-9020-eaca2cf6ce8c"), full_name: "oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone/oxz_crucible_1dfeb85c-baf9-49e2-9020-eaca2cf6ce8c" }
file = sled-storage/src/manager.rs:1331
|
| 553 | 2024-12-19T00:38:08.421Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 554 | 2024-12-19T00:38:08.421Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone/oxz_crucible_9785445e-9baa-4024-8a89-6b305fb3e36a"), full_name: "oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone/oxz_crucible_9785445e-9baa-4024-8a89-6b305fb3e36a" }
file = sled-storage/src/manager.rs:1331
|
| 555 | 2024-12-19T00:38:08.518Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 556 | 2024-12-19T00:38:08.518Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone"), full_name: "oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone" }
file = sled-storage/src/manager.rs:1331
|
| 557 | 2024-12-19T00:38:08.585Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 558 | 2024-12-19T00:38:08.585Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/cockroachdb" }
file = sled-storage/src/manager.rs:1331
|
| 559 | 2024-12-19T00:38:08.675Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 560 | 2024-12-19T00:38:08.675Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone"), full_name: "oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone" }
file = sled-storage/src/manager.rs:1331
|
| 561 | 2024-12-19T00:38:08.744Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 562 | 2024-12-19T00:38:08.744Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/debug"), full_name: "oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/debug" }
file = sled-storage/src/manager.rs:1331
|
| 563 | 2024-12-19T00:38:08.811Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 564 | 2024-12-19T00:38:08.811Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_external_dns_7b1c75dd-240b-4017-a160-edd46f1b3d00"), full_name: "oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_external_dns_7b1c75dd-240b-4017-a160-edd46f1b3d00" }
file = sled-storage/src/manager.rs:1331
|
| 565 | 2024-12-19T00:38:08.908Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 566 | 2024-12-19T00:38:08.908Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone/oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891"), full_name: "oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone/oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891" }
file = sled-storage/src/manager.rs:1331
|
| 567 | 2024-12-19T00:38:09.031Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 568 | 2024-12-19T00:38:09.031Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7"), full_name: "oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7" }
file = sled-storage/src/manager.rs:1331
|
| 569 | 2024-12-19T00:38:09.073Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)
file = sled-agent/src/probe_manager.rs:231
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 570 | 2024-12-19T00:38:09.134Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 571 | 2024-12-19T00:38:09.134Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/debug"), full_name: "oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/debug" }
file = sled-storage/src/manager.rs:1331
|
| 572 | 2024-12-19T00:38:09.204Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 573 | 2024-12-19T00:38:09.204Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_nexus_445c7eec-d66d-460b-9a24-27198fcf6179"), full_name: "oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_nexus_445c7eec-d66d-460b-9a24-27198fcf6179" }
file = sled-storage/src/manager.rs:1331
|
| 574 | 2024-12-19T00:38:09.306Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 575 | 2024-12-19T00:38:09.306Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone/oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba"), full_name: "oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone/oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba" }
file = sled-storage/src/manager.rs:1331
|
| 576 | 2024-12-19T00:38:09.407Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 577 | 2024-12-19T00:38:09.407Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f"), full_name: "oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f" }
file = sled-storage/src/manager.rs:1331
|
| 578 | 2024-12-19T00:38:09.506Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 579 | 2024-12-19T00:38:09.506Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0/crucible" }
file = sled-storage/src/manager.rs:1331
|
| 580 | 2024-12-19T00:38:09.595Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 581 | 2024-12-19T00:38:09.595Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7"), full_name: "oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7" }
file = sled-storage/src/manager.rs:1331
|
| 582 | 2024-12-19T00:38:09.701Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 583 | 2024-12-19T00:38:09.701Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: true, mountpoint: Path("/data"), full_name: "oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/internal_dns" }
file = sled-storage/src/manager.rs:1331
|
| 584 | 2024-12-19T00:38:09.794Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 585 | 2024-12-19T00:38:09.794Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_clickhouse_20d7411f-1498-441e-89c3-f4071a3d9fb5"), full_name: "oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_clickhouse_20d7411f-1498-441e-89c3-f4071a3d9fb5" }
file = sled-storage/src/manager.rs:1331
|
| 586 | 2024-12-19T00:38:09.891Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 587 | 2024-12-19T00:38:09.891Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone/oxz_crucible_1cc9eeb4-0d42-4938-8c8b-a21df5e97007"), full_name: "oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone/oxz_crucible_1cc9eeb4-0d42-4938-8c8b-a21df5e97007" }
file = sled-storage/src/manager.rs:1331
|
| 588 | 2024-12-19T00:38:09.903Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 12768160433890941050
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 589 | 2024-12-19T00:38:09.903Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 14686322292127562160
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 590 | 2024-12-19T00:38:09.989Z | INFO | SledAgent (StorageManager): Ensuring dataset
file = sled-storage/src/manager.rs:949
request = datasets_ensure
|
| 591 | 2024-12-19T00:38:09.989Z | INFO | SledAgent (StorageManager): ensure_dataset
config = SharedDatasetConfig { compression: Off, quota: None, reservation: None }
details = DatasetCreationDetails { zoned: false, mountpoint: Path("/pool/ext/c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_external_dns_77622b51-9ad8-477c-8380-8cae282506aa"), full_name: "oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_external_dns_77622b51-9ad8-477c-8380-8cae282506aa" }
file = sled-storage/src/manager.rs:1331
|
| 592 | 2024-12-19T00:38:10.075Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)
file = sled-agent/src/probe_manager.rs:231
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 593 | 2024-12-19T00:38:10.089Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/.omicron-datasets.json.tmp
file = common/src/ledger.rs:199
request = datasets_ensure
|
| 594 | 2024-12-19T00:38:10.093Z | INFO | SledAgent (StorageManager): Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/.omicron-datasets.json.tmp
file = common/src/ledger.rs:199
request = datasets_ensure
|
| 595 | 2024-12-19T00:38:10.093Z | INFO | SledAgent: datasets ensure: Updated storage
file = sled-agent/src/sled_agent.rs:864
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 596 | 2024-12-19T00:38:10.093Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 5797743
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:46149
req_id = e2721a10-c03a-4533-8649-4e91ea512385
response_code = 200
uri = /datasets
|
| 597 | 2024-12-19T00:38:10.116Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists - OK
file = sled-agent/src/services.rs:4292
|
| 598 | 2024-12-19T00:38:10.116Z | INFO | SledAgent (ServiceManager): Ensuring there is a default route
file = sled-agent/src/services.rs:4302
gateway = fd00:1122:3344:101::1
|
| 599 | 2024-12-19T00:38:10.123Z | INFO | SledAgent (ServiceManager): Default route already exists
file = sled-agent/src/services.rs:4316
gateway = fd00:1122:3344:101::1
|
| 600 | 2024-12-19T00:38:10.123Z | INFO | SledAgent (ServiceManager): configuring dendrite service
file = sled-agent/src/services.rs:4381
|
| 601 | 2024-12-19T00:38:10.133Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29 (service), filesystem_pool: Some(ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 8cda43db-36a7-4db9-aa46-224c2e7183f5 (service), filesystem_pool: Some(ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f (service), filesystem_pool: Some(ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }
file = sled-agent/src/rack_setup/service.rs:541
sled_address = [fd00:1122:3344:101::1]:12345
|
| 602 | 2024-12-19T00:38:10.133Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:34250
|
| 603 | 2024-12-19T00:38:10.133Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:603
|
| 604 | 2024-12-19T00:38:10.133Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1379
request = NewFilesystemRequest { dataset_id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 605 | 2024-12-19T00:38:10.170Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cda43db-36a7-4db9-aa46-224c2e7183f5, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:603
|
| 606 | 2024-12-19T00:38:10.170Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1379
request = NewFilesystemRequest { dataset_id: 8cda43db-36a7-4db9-aa46-224c2e7183f5, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 607 | 2024-12-19T00:38:10.203Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:603
|
| 608 | 2024-12-19T00:38:10.203Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1379
request = NewFilesystemRequest { dataset_id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 609 | 2024-12-19T00:38:10.239Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 610 | 2024-12-19T00:38:10.239Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 611 | 2024-12-19T00:38:10.239Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/omicron-zones.json
file = common/src/ledger.rs:188
|
| 612 | 2024-12-19T00:38:10.239Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 613 | 2024-12-19T00:38:10.279Z | INFO | SledAgent (ServiceManager): refreshed dendrite service with new configuration
file = sled-agent/src/services.rs:4447
|
| 614 | 2024-12-19T00:38:10.279Z | INFO | SledAgent (ServiceManager): configuring lldp service
file = sled-agent/src/services.rs:4474
|
| 615 | 2024-12-19T00:38:10.290Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 616 | 2024-12-19T00:38:10.290Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 617 | 2024-12-19T00:38:10.290Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 618 | 2024-12-19T00:38:10.325Z | INFO | SledAgent (ServiceManager): refreshed lldpd service with new configuration
file = sled-agent/src/services.rs:4487
|
| 619 | 2024-12-19T00:38:10.325Z | INFO | SledAgent (ServiceManager): configuring MGS service
file = sled-agent/src/services.rs:4333
|
| 620 | 2024-12-19T00:38:10.351Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f
file = illumos-utils/src/zone.rs:325
|
| 621 | 2024-12-19T00:38:10.355Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5
file = illumos-utils/src/zone.rs:325
|
| 622 | 2024-12-19T00:38:10.362Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29
file = illumos-utils/src/zone.rs:325
|
| 623 | 2024-12-19T00:38:10.373Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f
file = illumos-utils/src/zone.rs:363
|
| 624 | 2024-12-19T00:38:10.378Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5
file = illumos-utils/src/zone.rs:363
|
| 625 | 2024-12-19T00:38:10.381Z | INFO | SledAgent (ServiceManager): refreshed MGS service with new configuration
file = sled-agent/src/services.rs:4375
|
| 626 | 2024-12-19T00:38:10.381Z | INFO | SledAgent (ServiceManager): configuring wicketd
file = sled-agent/src/services.rs:4454
rack_subnet = fd00:1122:3344::
|
| 627 | 2024-12-19T00:38:10.384Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29
file = illumos-utils/src/zone.rs:363
|
| 628 | 2024-12-19T00:38:10.404Z | INFO | SledAgent (ServiceManager): refreshed wicketd service with new configuration
file = sled-agent/src/services.rs:4465
|
| 629 | 2024-12-19T00:38:10.404Z | INFO | SledAgent (ServiceManager): configuring mgd service
file = sled-agent/src/services.rs:4507
|
| 630 | 2024-12-19T00:38:10.491Z | INFO | SledAgent (ServiceManager): refreshed mgd service with new configuration
file = sled-agent/src/services.rs:4539
|
| 631 | 2024-12-19T00:38:10.491Z | INFO | SledAgent (ServiceManager): configuring mg-ddm service
file = sled-agent/src/services.rs:4545
|
| 632 | 2024-12-19T00:38:10.604Z | INFO | SledAgent (ServiceManager): refreshed mg-ddm service with new configuration
file = sled-agent/src/services.rs:4586
|
| 633 | 2024-12-19T00:38:10.604Z | INFO | SledAgent (ServiceManager): Determining physical location of our switch zone at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:370
|
| 634 | 2024-12-19T00:38:10.642Z | INFO | SledAgent (ServiceManager): Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:420
|
| 635 | 2024-12-19T00:38:10.642Z | WARN | SledAgent (NexusNotifierTask): Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e358e662-7828-4f7f-85a4-21e089483ee6)
file = sled-agent/src/nexus.rs:527
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 636 | 2024-12-19T00:38:10.683Z | INFO | SledAgent (ServiceManager): Building Port Configuration
file = sled-agent/src/bootstrap/early_networking.rs:717
|
| 637 | 2024-12-19T00:38:10.683Z | INFO | SledAgent (ServiceManager): Checking dendrite uptime
file = sled-agent/src/bootstrap/early_networking.rs:762
|
| 638 | 2024-12-19T00:38:10.684Z | INFO | SledAgent (ServiceManager): Dendrite online
file = sled-agent/src/bootstrap/early_networking.rs:765
uptime = 1
|
| 639 | 2024-12-19T00:38:10.684Z | INFO | SledAgent (ServiceManager): Configuring default uplink on switch
config = PortSettings {\n links: {\n "0": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: Some(\n None,\n ),\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n tx_eq: None,\n },\n },\n },\n}
file = sled-agent/src/bootstrap/early_networking.rs:444
|
| 640 | 2024-12-19T00:38:10.731Z | INFO | SledAgent (ServiceManager): ensuring scrimlet uplinks
file = sled-agent/src/services.rs:4119
|
| 641 | 2024-12-19T00:38:10.788Z | INFO | SledAgent (StorageManager): Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }
file = sled-storage/src/manager.rs:603
|
| 642 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 643 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-0" }
|
| 644 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 645 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-M2", serial: "synthetic-serial-1" }
|
| 646 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 647 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-0" }
|
| 648 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 649 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-1" }
|
| 650 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 651 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-2" }
|
| 652 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 653 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-3" }
|
| 654 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 655 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-4" }
|
| 656 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 657 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-5" }
|
| 658 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 659 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-6" }
|
| 660 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 661 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-7" }
|
| 662 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Removing disk
file = sled-storage/src/resources.rs:529
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 663 | 2024-12-19T00:38:10.789Z | INFO | SledAgent (StorageResources): Not removing synthetic disk
file = sled-storage/src/resources.rs:550
identity = DiskIdentity { vendor: "synthetic-vendor", model: "synthetic-model-U2", serial: "synthetic-serial-8" }
|
| 664 | 2024-12-19T00:38:11.077Z | ERRO | SledAgent (ProbeManager): get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)
file = sled-agent/src/probe_manager.rs:231
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 665 | 2024-12-19T00:38:11.268Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 14076285699142498485
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 666 | 2024-12-19T00:38:11.268Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 15791617063128661124
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 667 | 2024-12-19T00:38:12.015Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService3'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:1::2'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:1::1'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value='[fd00:1122:3344:1::1]:5353'/>
<propval type="astring" name="dns_address" value='[fd00:1122:3344:1::1]:53'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 668 | 2024-12-19T00:38:12.015Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }
|
| 669 | 2024-12-19T00:38:12.494Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService1'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:3::2'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:3::1'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value='[fd00:1122:3344:3::1]:5353'/>
<propval type="astring" name="dns_address" value='[fd00:1122:3344:3::1]:53'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 670 | 2024-12-19T00:38:12.971Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }
|
| 671 | 2024-12-19T00:38:12.971Z | INFO | SledAgent (ServiceManager): Profile for oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService2'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:2::2'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:2::1'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/internal_dns">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="http_address" value='[fd00:1122:3344:2::1]:5353'/>
<propval type="astring" name="dns_address" value='[fd00:1122:3344:2::1]:53'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="false" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 672 | 2024-12-19T00:38:12.971Z | INFO | SledAgent (ServiceManager): Booting oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29
|
| 673 | 2024-12-19T00:38:12.975Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:86
prefix = Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }
|
| 674 | 2024-12-19T00:38:12.975Z | INFO | SledAgent (ServiceManager): Booting oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f
|
| 675 | 2024-12-19T00:38:12.979Z | INFO | SledAgent (ServiceManager): Booting oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5
|
| 676 | 2024-12-19T00:38:13.106Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29") failed: Property not found. retry in 34.28747ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29
|
| 677 | 2024-12-19T00:38:13.112Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f") failed: Property not found. retry in 43.736626ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f
|
| 678 | 2024-12-19T00:38:13.118Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5") failed: Property not found. retry in 60.921211ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5
|
| 679 | 2024-12-19T00:38:13.148Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29") failed: Property not found. retry in 126.282121ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29
|
| 680 | 2024-12-19T00:38:13.165Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f") failed: Property not found. retry in 97.030703ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f
|
| 681 | 2024-12-19T00:38:13.331Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5") failed: Property not found. retry in 109.643284ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5
|
| 682 | 2024-12-19T00:38:13.503Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f") failed: Property not found. retry in 177.020697ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f
|
| 683 | 2024-12-19T00:38:13.527Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5") failed: Property not found. retry in 112.686102ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5
|
| 684 | 2024-12-19T00:38:13.604Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29") failed: Property not found. retry in 261.70165ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29
|
| 685 | 2024-12-19T00:38:13.714Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5") failed: Property not found. retry in 418.659812ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5
|
| 686 | 2024-12-19T00:38:13.737Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f") failed: Property not found. retry in 584.008776ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f
|
| 687 | 2024-12-19T00:38:13.893Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29") failed: Property not found. retry in 548.140796ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29
|
| 688 | 2024-12-19T00:38:14.143Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5") failed: Property not found. retry in 993.42979ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5
|
| 689 | 2024-12-19T00:38:14.331Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f") failed: Property not found. retry in 756.716093ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f
|
| 690 | 2024-12-19T00:38:14.452Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29") failed: Property not found. retry in 511.748683ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29
|
| 691 | 2024-12-19T00:38:14.973Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29") failed: Property not found. retry in 1.421549307s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29
|
| 692 | 2024-12-19T00:38:15.098Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f") failed: Property not found. retry in 1.418229717s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f
|
| 693 | 2024-12-19T00:38:15.146Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5") failed: Property not found. retry in 977.984982ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5
|
| 694 | 2024-12-19T00:38:16.133Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5") failed: Property not found. retry in 691.403821ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5
|
| 695 | 2024-12-19T00:38:16.406Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29") failed: Property not found. retry in 633.582444ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29
|
| 696 | 2024-12-19T00:38:16.529Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f") failed: Property not found. retry in 1.353355823s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f
|
| 697 | 2024-12-19T00:38:16.835Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5") failed: Property not found. retry in 1.335604394s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5
|
| 698 | 2024-12-19T00:38:17.048Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29") failed: Property not found. retry in 758.311467ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29
|
| 699 | 2024-12-19T00:38:17.816Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29") failed: Property not found. retry in 1.094720197s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29
|
| 700 | 2024-12-19T00:38:17.890Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f") failed: Property not found. retry in 716.258683ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f
|
| 701 | 2024-12-19T00:38:18.180Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5") failed: Property not found. retry in 728.140452ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5
|
| 702 | 2024-12-19T00:38:18.615Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f") failed: Property not found. retry in 584.775374ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f
|
| 703 | 2024-12-19T00:38:18.918Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5") failed: Property not found. retry in 915.913916ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5
|
| 704 | 2024-12-19T00:38:18.929Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29") failed: Property not found. retry in 897.230337ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29
|
| 705 | 2024-12-19T00:38:19.210Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f") failed: Property not found. retry in 1.072919887s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f
|
| 706 | 2024-12-19T00:38:19.905Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 12768160433890941050
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 707 | 2024-12-19T00:38:19.905Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 14686322292127562160
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 708 | 2024-12-19T00:38:20.303Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3310
zone = oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f
|
| 709 | 2024-12-19T00:38:20.303Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3310
zone = oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5
|
| 710 | 2024-12-19T00:38:20.303Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3310
zone = oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29
|
| 711 | 2024-12-19T00:38:20.303Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 712 | 2024-12-19T00:38:20.303Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 713 | 2024-12-19T00:38:20.303Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 10169868
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:34250
req_id = c988513f-a8ce-42a7-9424-6aebb2b3cd10
response_code = 204
uri = /omicron-zones
|
| 714 | 2024-12-19T00:38:20.305Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:1::1]:5353
file = sled-agent/src/rack_setup/service.rs:658
|
| 715 | 2024-12-19T00:38:21.270Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 14076285699142498485
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 716 | 2024-12-19T00:38:21.271Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 15791617063128661124
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 717 | 2024-12-19T00:38:21.392Z | WARN | SledAgent (RSS): failed to write DNS configuration (will retry in 70.332979ms)
dns_config_addr = [fd00:1122:3344:1::1]:5353
error_message = Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config)
file = sled-agent/src/rack_setup/service.rs:678
|
| 718 | 2024-12-19T00:38:21.463Z | WARN | SledAgent (RSS): failed to write DNS configuration (will retry in 144.342656ms)
dns_config_addr = [fd00:1122:3344:1::1]:5353
error_message = Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config)
file = sled-agent/src/rack_setup/service.rs:678
|
| 719 | 2024-12-19T00:38:21.659Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:2::1]:5353
file = sled-agent/src/rack_setup/service.rs:658
|
| 720 | 2024-12-19T00:38:21.738Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:3::1]:5353
file = sled-agent/src/rack_setup/service.rs:658
|
| 721 | 2024-12-19T00:38:21.819Z | INFO | SledAgent (RSS): Configured all DNS servers
file = sled-agent/src/rack_setup/service.rs:694
|
| 722 | 2024-12-19T00:38:21.819Z | INFO | SledAgent (RSS): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:264
|
| 723 | 2024-12-19T00:38:21.852Z | INFO | SledAgent (RSS): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:281
|
| 724 | 2024-12-19T00:38:21.854Z | INFO | SledAgent (RSS): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:306
|
| 725 | 2024-12-19T00:38:21.854Z | INFO | SledAgent (RSS): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:198
|
| 726 | 2024-12-19T00:38:21.885Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29 (service), filesystem_pool: Some(ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 8cda43db-36a7-4db9-aa46-224c2e7183f5 (service), filesystem_pool: Some(ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f (service), filesystem_pool: Some(ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 931fa059-cc81-4b86-be35-dae0f4c5b891 (service), filesystem_pool: Some(ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a3b3f151-c7d2-4d8a-8722-84fd46ba2ae6, kind: Service { id: 931fa059-cc81-4b86-be35-dae0f4c5b891 }, name: Name("ntp-931fa059-cc81-4b86-be35-dae0f4c5b891"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 229])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }
file = sled-agent/src/rack_setup/service.rs:541
sled_address = [fd00:1122:3344:101::1]:12345
|
| 727 | 2024-12-19T00:38:21.885Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:40568
|
| 728 | 2024-12-19T00:38:21.885Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:603
|
| 729 | 2024-12-19T00:38:21.885Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1379
request = NewFilesystemRequest { dataset_id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 730 | 2024-12-19T00:38:21.931Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cda43db-36a7-4db9-aa46-224c2e7183f5, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:603
|
| 731 | 2024-12-19T00:38:21.931Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1379
request = NewFilesystemRequest { dataset_id: 8cda43db-36a7-4db9-aa46-224c2e7183f5, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 732 | 2024-12-19T00:38:21.975Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:603
|
| 733 | 2024-12-19T00:38:21.975Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1379
request = NewFilesystemRequest { dataset_id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 734 | 2024-12-19T00:38:22.019Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 735 | 2024-12-19T00:38:22.019Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 736 | 2024-12-19T00:38:22.020Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 737 | 2024-12-19T00:38:22.020Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 738 | 2024-12-19T00:38:22.031Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:264
|
| 739 | 2024-12-19T00:38:22.063Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:281
|
| 740 | 2024-12-19T00:38:22.064Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:306
|
| 741 | 2024-12-19T00:38:22.064Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:198
|
| 742 | 2024-12-19T00:38:22.094Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:447
port = Port { inner: PortInner(PortData { name: "opte0", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 219, 229]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }
|
| 743 | 2024-12-19T00:38:22.094Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1263
zone_type = boundary_ntp
|
| 744 | 2024-12-19T00:38:22.094Z | INFO | SledAgent (ServiceManager): no nat entry found for: 10.151.2.169
file = clients/dpd-client/src/lib.rs:83
|
| 745 | 2024-12-19T00:38:22.094Z | INFO | SledAgent (ServiceManager): creating nat entry for: 10.151.2.169
file = clients/dpd-client/src/lib.rs:90
|
| 746 | 2024-12-19T00:38:22.095Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: 10.151.2.169
file = clients/dpd-client/src/lib.rs:118
|
| 747 | 2024-12-19T00:38:22.095Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 748 | 2024-12-19T00:38:22.119Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891
file = illumos-utils/src/zone.rs:325
|
| 749 | 2024-12-19T00:38:22.137Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891
file = illumos-utils/src/zone.rs:363
|
| 750 | 2024-12-19T00:38:22.755Z | INFO | SledAgent (ServiceManager): Profile for oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService4'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::1b'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/chrony-setup">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="allow" value='fd00:1122:3344:100::/56'/>
<propval type="boolean" name="boundary" value='true'/>
<propval type="astring" name="boundary_pool" value='boundary-ntp.control-plane.oxide.internal'/>
<propval type="astring" name="server" value='0.pool.ntp.org'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="1.1.1.1" />
<value_node value="9.9.9.9" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/ntp">
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/opte-interface-setup">
<property_group type="application" name="config">
<propval type="astring" name="interface" value='opte0'/>
<propval type="astring" name="gateway" value='172.30.3.1'/>
<propval type="astring" name="ip" value='172.30.3.5'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 751 | 2024-12-19T00:38:22.755Z | INFO | SledAgent (ServiceManager): Booting oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891
|
| 752 | 2024-12-19T00:38:22.892Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891") failed: Property not found. retry in 59.18818ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891
|
| 753 | 2024-12-19T00:38:22.962Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891") failed: Property not found. retry in 128.323297ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891
|
| 754 | 2024-12-19T00:38:23.101Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891") failed: Property not found. retry in 127.672678ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891
|
| 755 | 2024-12-19T00:38:23.239Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891") failed: Property not found. retry in 436.663855ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891
|
| 756 | 2024-12-19T00:38:23.686Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891") failed: Property not found. retry in 782.346022ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891
|
| 757 | 2024-12-19T00:38:24.478Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891") failed: Property not found. retry in 1.206492554s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891
|
| 758 | 2024-12-19T00:38:25.694Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891") failed: Property not found. retry in 704.855546ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891
|
| 759 | 2024-12-19T00:38:26.409Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891") failed: Property not found. retry in 590.799432ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891
|
| 760 | 2024-12-19T00:38:27.009Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891") failed: Property not found. retry in 1.006614826s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891
|
| 761 | 2024-12-19T00:38:28.025Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891") failed: Property not found. retry in 890.919471ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891
|
| 762 | 2024-12-19T00:38:28.935Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3310
zone = oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891
|
| 763 | 2024-12-19T00:38:28.935Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 764 | 2024-12-19T00:38:28.936Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 765 | 2024-12-19T00:38:28.936Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 7051203
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:40568
req_id = 5e22cc44-6d19-4e36-bdaa-05c64c48703d
response_code = 204
uri = /omicron-zones
|
| 766 | 2024-12-19T00:38:28.938Z | INFO | SledAgent (RSS): Waiting for rack time synchronization
file = sled-agent/src/rack_setup/service.rs:735
|
| 767 | 2024-12-19T00:38:28.968Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 768 | 2024-12-19T00:38:28.968Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:36488
|
| 769 | 2024-12-19T00:38:28.978Z | ERRO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3900
|
| 770 | 2024-12-19T00:38:28.978Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 9330
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:36488
req_id = 556afa6e-37b0-40f5-97e8-15bf2eb5b903
response_code = 500
uri = /timesync
|
| 771 | 2024-12-19T00:38:28.979Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 772 | 2024-12-19T00:38:29.141Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 773 | 2024-12-19T00:38:29.141Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:57119
|
| 774 | 2024-12-19T00:38:29.863Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 13115064489608424063
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 775 | 2024-12-19T00:38:29.866Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 11430979086984911751
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 776 | 2024-12-19T00:38:29.906Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 12768160433890941050
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 777 | 2024-12-19T00:38:29.906Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 14686322292127562160
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 778 | 2024-12-19T00:38:30.307Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 17942048789459529530
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 779 | 2024-12-19T00:38:31.272Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 14076285699142498485
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 780 | 2024-12-19T00:38:31.272Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 15791617063128661124
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 781 | 2024-12-19T00:38:33.151Z | ERRO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3900
|
| 782 | 2024-12-19T00:38:33.151Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 4009265
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:57119
req_id = b39957e3-c4d1-4487-b6d9-2a7ebd5c0461
response_code = 500
uri = /timesync
|
| 783 | 2024-12-19T00:38:33.153Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 784 | 2024-12-19T00:38:33.357Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 785 | 2024-12-19T00:38:33.357Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:49077
|
| 786 | 2024-12-19T00:38:33.365Z | ERRO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3900
|
| 787 | 2024-12-19T00:38:33.365Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 7834
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:49077
req_id = 2662d1fa-bc8c-4b22-8b9b-e5bd0ce7c032
response_code = 500
uri = /timesync
|
| 788 | 2024-12-19T00:38:33.368Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 789 | 2024-12-19T00:38:33.511Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 790 | 2024-12-19T00:38:33.511Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:63998
|
| 791 | 2024-12-19T00:38:37.511Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 244.175567ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:420
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 792 | 2024-12-19T00:38:37.522Z | ERRO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3900
|
| 793 | 2024-12-19T00:38:37.522Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 4007234
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:63998
req_id = 25fae244-fe9a-48fa-a3d0-4b764c11320f
response_code = 500
uri = /timesync
|
| 794 | 2024-12-19T00:38:37.524Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 795 | 2024-12-19T00:38:37.677Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 796 | 2024-12-19T00:38:37.678Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:54361
|
| 797 | 2024-12-19T00:38:37.687Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 9248
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:54361
req_id = 462f45ae-fc51-4074-a6a9-fa5877f8e614
response_code = 200
uri = /timesync
|
| 798 | 2024-12-19T00:38:37.689Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.085487872 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 799 | 2024-12-19T00:38:37.689Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 800 | 2024-12-19T00:38:37.948Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 801 | 2024-12-19T00:38:37.948Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:58312
|
| 802 | 2024-12-19T00:38:37.957Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 8887
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:58312
req_id = 082c9671-cd28-48b7-a3e2-9a8f02db7361
response_code = 200
uri = /timesync
|
| 803 | 2024-12-19T00:38:37.959Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.085169524 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 804 | 2024-12-19T00:38:37.959Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 805 | 2024-12-19T00:38:38.319Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 806 | 2024-12-19T00:38:38.320Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:44376
|
| 807 | 2024-12-19T00:38:38.328Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 7820
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:44376
req_id = ace71a56-55be-48e6-8e39-4e1bf9be8489
response_code = 200
uri = /timesync
|
| 808 | 2024-12-19T00:38:38.329Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.084168293 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 809 | 2024-12-19T00:38:38.329Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 810 | 2024-12-19T00:38:38.629Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 811 | 2024-12-19T00:38:38.629Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:64393
|
| 812 | 2024-12-19T00:38:38.637Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 8052
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:64393
req_id = faffde44-d579-4e50-b982-73fff477dabd
response_code = 200
uri = /timesync
|
| 813 | 2024-12-19T00:38:38.639Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.083331518 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 814 | 2024-12-19T00:38:38.639Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 815 | 2024-12-19T00:38:38.949Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 5828729199679799402
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 816 | 2024-12-19T00:38:38.949Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 9931914467758814075
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 817 | 2024-12-19T00:38:39.056Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 818 | 2024-12-19T00:38:39.056Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:40178
|
| 819 | 2024-12-19T00:38:39.065Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 8682
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:40178
req_id = 02dcb4d5-6576-4019-88a4-fc3068fdccd8
response_code = 200
uri = /timesync
|
| 820 | 2024-12-19T00:38:39.067Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568718.4096134, correction: 0.082337789 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 821 | 2024-12-19T00:38:39.067Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 822 | 2024-12-19T00:38:39.347Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 823 | 2024-12-19T00:38:39.347Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:61026
|
| 824 | 2024-12-19T00:38:39.355Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 7610
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:61026
req_id = 06c1b513-2b08-47b1-b2d3-723a015b42ef
response_code = 200
uri = /timesync
|
| 825 | 2024-12-19T00:38:39.359Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568718.4096134, correction: 0.081554256 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 826 | 2024-12-19T00:38:39.359Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 827 | 2024-12-19T00:38:39.853Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 828 | 2024-12-19T00:38:39.853Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:54133
|
| 829 | 2024-12-19T00:38:39.861Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 8257
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:54133
req_id = ce5f2cdf-18ce-447c-8b00-60ce112387ea
response_code = 200
uri = /timesync
|
| 830 | 2024-12-19T00:38:39.863Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568718.8964703, correction: 0.079777628 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 831 | 2024-12-19T00:38:39.863Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 832 | 2024-12-19T00:38:39.876Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 13115064489608424063
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 833 | 2024-12-19T00:38:39.879Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 11430979086984911751
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 834 | 2024-12-19T00:38:39.918Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 12768160433890941050
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 835 | 2024-12-19T00:38:39.918Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 14686322292127562160
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 836 | 2024-12-19T00:38:40.319Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 17942048789459529530
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 837 | 2024-12-19T00:38:40.347Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:62257
|
| 838 | 2024-12-19T00:38:40.347Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 97
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:62257
req_id = f61ac20d-893c-4afa-950e-55d396f5acf3
response_code = 200
uri = /baseboard
|
| 839 | 2024-12-19T00:38:40.728Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 840 | 2024-12-19T00:38:40.728Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:60628
|
| 841 | 2024-12-19T00:38:40.738Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 9660
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:60628
req_id = 9a14dc66-79c9-46cc-b153-89d18f6518ef
response_code = 200
uri = /timesync
|
| 842 | 2024-12-19T00:38:40.740Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568718.8964703, correction: 0.077410877 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 843 | 2024-12-19T00:38:40.740Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 844 | 2024-12-19T00:38:41.290Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 14076285699142498485
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 845 | 2024-12-19T00:38:41.290Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 15791617063128661124
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 846 | 2024-12-19T00:38:41.576Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 847 | 2024-12-19T00:38:41.576Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:61034
|
| 848 | 2024-12-19T00:38:41.585Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 8944
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:61034
req_id = dc954721-f2bc-4b29-904d-b234b29f8324
response_code = 200
uri = /timesync
|
| 849 | 2024-12-19T00:38:41.589Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568720.7604134, correction: 0.075146243 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 850 | 2024-12-19T00:38:41.589Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 851 | 2024-12-19T00:38:42.499Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 852 | 2024-12-19T00:38:42.499Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:62756
|
| 853 | 2024-12-19T00:38:42.508Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 8709
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:62756
req_id = 950cc70f-f9a5-4335-adec-5c8ee0030474
response_code = 200
uri = /timesync
|
| 854 | 2024-12-19T00:38:42.510Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568721.2434585, correction: 0.072281636 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 855 | 2024-12-19T00:38:42.510Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 856 | 2024-12-19T00:38:44.095Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 857 | 2024-12-19T00:38:44.095Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:42808
|
| 858 | 2024-12-19T00:38:44.104Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 8518
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:42808
req_id = 82d44f9d-3dd4-490d-8a79-64f1eefb94c2
response_code = 200
uri = /timesync
|
| 859 | 2024-12-19T00:38:44.107Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568723.2554588, correction: 0.067650191 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 860 | 2024-12-19T00:38:44.107Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 861 | 2024-12-19T00:38:45.946Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 862 | 2024-12-19T00:38:45.946Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:40085
|
| 863 | 2024-12-19T00:38:45.955Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 8941
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:40085
req_id = f4409bdb-8ba0-4283-8a1e-a5dcea876c1f
response_code = 200
uri = /timesync
|
| 864 | 2024-12-19T00:38:45.957Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568725.2572372, correction: 0.062688097 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 865 | 2024-12-19T00:38:45.957Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 866 | 2024-12-19T00:38:47.058Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 867 | 2024-12-19T00:38:47.058Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:38295
|
| 868 | 2024-12-19T00:38:47.067Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 8808
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:38295
req_id = b482f227-909b-4d3c-8ccf-87ae0e89d471
response_code = 200
uri = /timesync
|
| 869 | 2024-12-19T00:38:47.069Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568726.7274144, correction: 0.059760634 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 870 | 2024-12-19T00:38:47.069Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 871 | 2024-12-19T00:38:48.979Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 5828729199679799402
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 872 | 2024-12-19T00:38:48.979Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 9931914467758814075
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 873 | 2024-12-19T00:38:49.745Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 874 | 2024-12-19T00:38:49.745Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:51907
|
| 875 | 2024-12-19T00:38:49.754Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 9038
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:51907
req_id = 7d7eb8f8-4990-49e4-a34e-349281f5582e
response_code = 200
uri = /timesync
|
| 876 | 2024-12-19T00:38:49.758Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568728.5917926, correction: 0.052509069 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 877 | 2024-12-19T00:38:49.758Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:765
|
| 878 | 2024-12-19T00:38:49.905Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 13115064489608424063
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 879 | 2024-12-19T00:38:49.908Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 11430979086984911751
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 880 | 2024-12-19T00:38:49.947Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 12768160433890941050
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 881 | 2024-12-19T00:38:49.947Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 14686322292127562160
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 882 | 2024-12-19T00:38:50.348Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 17942048789459529530
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 883 | 2024-12-19T00:38:51.320Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 14076285699142498485
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 884 | 2024-12-19T00:38:51.320Z | ERRO | SledAgent (kstat-sampler-worker): failed to sample kstat target, requeueing
error = NoSuchKstat
file = oximeter/instruments/src/kstat/sampler.rs:496
id = 15791617063128661124
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 885 | 2024-12-19T00:38:52.448Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:715
|
| 886 | 2024-12-19T00:38:52.448Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:62066
|
| 887 | 2024-12-19T00:38:52.470Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 21503
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:62066
req_id = cf2a67ab-faeb-422c-ac85-dd4bb7d5e796
response_code = 200
uri = /timesync
|
| 888 | 2024-12-19T00:38:52.474Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568730.42558, correction: 0.045051727 }
file = sled-agent/src/rack_setup/service.rs:743
|
| 889 | 2024-12-19T00:38:52.474Z | INFO | SledAgent (RSS): Finished setting up Internal DNS and NTP
file = sled-agent/src/rack_setup/service.rs:1392
|
| 890 | 2024-12-19T00:38:52.504Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29 (service), filesystem_pool: Some(ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 8cda43db-36a7-4db9-aa46-224c2e7183f5 (service), filesystem_pool: Some(ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f (service), filesystem_pool: Some(ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 931fa059-cc81-4b86-be35-dae0f4c5b891 (service), filesystem_pool: Some(ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a3b3f151-c7d2-4d8a-8722-84fd46ba2ae6, kind: Service { id: 931fa059-cc81-4b86-be35-dae0f4c5b891 }, name: Name("ntp-931fa059-cc81-4b86-be35-dae0f4c5b891"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 229])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 05bfdf27-226d-408e-8634-4cf7d9392bcf (service), filesystem_pool: Some(ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External } } } }, OmicronZoneConfig { id: a52f73c1-a1a5-4560-812f-18319626624c (service), filesystem_pool: Some(ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3cc04303-718d-447d-bc89-3a4154b619d7 (service), filesystem_pool: Some(ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 57ed5e38-59cc-4e8b-8338-2dceef12fef7 (service), filesystem_pool: Some(ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External } } } }, OmicronZoneConfig { id: 5c749b5a-3b6c-44a8-9cb5-f97707372cba (service), filesystem_pool: Some(ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:541
sled_address = [fd00:1122:3344:101::1]:12345
|
| 891 | 2024-12-19T00:38:52.504Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:64045
|
| 892 | 2024-12-19T00:38:52.505Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:603
|
| 893 | 2024-12-19T00:38:52.505Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1379
request = NewFilesystemRequest { dataset_id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 894 | 2024-12-19T00:38:52.543Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cda43db-36a7-4db9-aa46-224c2e7183f5, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:603
|
| 895 | 2024-12-19T00:38:52.543Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1379
request = NewFilesystemRequest { dataset_id: 8cda43db-36a7-4db9-aa46-224c2e7183f5, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 896 | 2024-12-19T00:38:52.580Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, responder: ... })
file = sled-storage/src/manager.rs:603
|
| 897 | 2024-12-19T00:38:52.580Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1379
request = NewFilesystemRequest { dataset_id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, responder: ... }
|
| 898 | 2024-12-19T00:38:52.614Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 05bfdf27-226d-408e-8634-4cf7d9392bcf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:603
|
| 899 | 2024-12-19T00:38:52.614Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1379
request = NewFilesystemRequest { dataset_id: 05bfdf27-226d-408e-8634-4cf7d9392bcf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 900 | 2024-12-19T00:38:52.653Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a52f73c1-a1a5-4560-812f-18319626624c, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:603
|
| 901 | 2024-12-19T00:38:52.653Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1379
request = NewFilesystemRequest { dataset_id: a52f73c1-a1a5-4560-812f-18319626624c, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 902 | 2024-12-19T00:38:52.689Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cc04303-718d-447d-bc89-3a4154b619d7, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:603
|
| 903 | 2024-12-19T00:38:52.689Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1379
request = NewFilesystemRequest { dataset_id: 3cc04303-718d-447d-bc89-3a4154b619d7, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 904 | 2024-12-19T00:38:52.726Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 57ed5e38-59cc-4e8b-8338-2dceef12fef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:603
|
| 905 | 2024-12-19T00:38:52.726Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1379
request = NewFilesystemRequest { dataset_id: 57ed5e38-59cc-4e8b-8338-2dceef12fef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 906 | 2024-12-19T00:38:52.761Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c749b5a-3b6c-44a8-9cb5-f97707372cba, dataset_name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Cockroach }, responder: ... })
file = sled-storage/src/manager.rs:603
|
| 907 | 2024-12-19T00:38:52.761Z | INFO | SledAgent (StorageManager): add_dataset
file = sled-storage/src/manager.rs:1379
request = NewFilesystemRequest { dataset_id: 5c749b5a-3b6c-44a8-9cb5-f97707372cba, dataset_name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Cockroach }, responder: ... }
|
| 908 | 2024-12-19T00:38:52.797Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 909 | 2024-12-19T00:38:52.797Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 910 | 2024-12-19T00:38:52.797Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/omicron-zones.json
file = common/src/ledger.rs:177
|
| 911 | 2024-12-19T00:38:52.797Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 912 | 2024-12-19T00:38:52.834Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 312.38578ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:420
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 913 | 2024-12-19T00:38:52.878Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 914 | 2024-12-19T00:38:52.878Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 915 | 2024-12-19T00:38:52.878Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 916 | 2024-12-19T00:38:52.878Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 917 | 2024-12-19T00:38:52.878Z | INFO | SledAgent (StorageManager): Received GetLatestResources(...)
file = sled-storage/src/manager.rs:603
|
| 918 | 2024-12-19T00:38:52.967Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
file = illumos-utils/src/zone.rs:325
|
| 919 | 2024-12-19T00:38:52.972Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
file = illumos-utils/src/zone.rs:325
|
| 920 | 2024-12-19T00:38:52.976Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
file = illumos-utils/src/zone.rs:325
|
| 921 | 2024-12-19T00:38:52.980Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
file = illumos-utils/src/zone.rs:325
|
| 922 | 2024-12-19T00:38:52.984Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
file = illumos-utils/src/zone.rs:325
|
| 923 | 2024-12-19T00:38:52.989Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
file = illumos-utils/src/zone.rs:363
|
| 924 | 2024-12-19T00:38:52.994Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
file = illumos-utils/src/zone.rs:363
|
| 925 | 2024-12-19T00:38:52.998Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
file = illumos-utils/src/zone.rs:363
|
| 926 | 2024-12-19T00:38:53.002Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
file = illumos-utils/src/zone.rs:363
|
| 927 | 2024-12-19T00:38:53.006Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
file = illumos-utils/src/zone.rs:363
|
| 928 | 2024-12-19T00:38:55.791Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService5'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::3'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::3]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='05bfdf27-226d-408e-8634-4cf7d9392bcf'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::3]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::3]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 929 | 2024-12-19T00:38:55.791Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 930 | 2024-12-19T00:38:55.837Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService8'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::4'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::4]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='a52f73c1-a1a5-4560-812f-18319626624c'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::4]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::4]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 931 | 2024-12-19T00:38:55.837Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 932 | 2024-12-19T00:38:55.895Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService9'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::6'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::6]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='57ed5e38-59cc-4e8b-8338-2dceef12fef7'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::6]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::6]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 933 | 2024-12-19T00:38:55.895Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 934 | 2024-12-19T00:38:55.914Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService7'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::5'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::5]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='3cc04303-718d-447d-bc89-3a4154b619d7'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::5]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::5]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 935 | 2024-12-19T00:38:55.914Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7 zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 936 | 2024-12-19T00:38:55.920Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value='oxControlService6'/>
<propval type="astring" name="gateway" value='fd00:1122:3344:101::1'/>
<propval type="astring" name="static_addr" value='fd00:1122:3344:101::7'/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value='[fd00:1122:3344:101::7]:32221'/>
<propval type="astring" name="store" value='/data'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="oxide/cockroach-admin">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="zone_id" value='5c749b5a-3b6c-44a8-9cb5-f97707372cba'/>
<propval type="astring" name="cockroach_address" value='[fd00:1122:3344:101::7]:32221'/>
<propval type="astring" name="http_address" value='[fd00:1122:3344:101::7]:32222'/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:1::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:3::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/dns/client">
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 937 | 2024-12-19T00:38:55.920Z | INFO | SledAgent (ServiceManager): Booting oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba zone
file = illumos-utils/src/running_zone.rs:488
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 938 | 2024-12-19T00:38:55.946Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 56.339247ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 939 | 2024-12-19T00:38:55.995Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 50.391227ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 940 | 2024-12-19T00:38:56.011Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 68.782609ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 941 | 2024-12-19T00:38:56.061Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 73.559923ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 942 | 2024-12-19T00:38:56.085Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 50.685773ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 943 | 2024-12-19T00:38:56.233Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 288.246784ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 944 | 2024-12-19T00:38:56.267Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 159.618055ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 945 | 2024-12-19T00:38:56.282Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 56.944482ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 946 | 2024-12-19T00:38:56.305Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 62.227662ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 947 | 2024-12-19T00:38:56.336Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 66.237825ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 948 | 2024-12-19T00:38:56.362Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 212.397393ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 949 | 2024-12-19T00:38:56.389Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 99.282639ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 950 | 2024-12-19T00:38:56.446Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 82.787028ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 951 | 2024-12-19T00:38:56.582Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 576.342207ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 952 | 2024-12-19T00:38:56.611Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 492.094388ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 953 | 2024-12-19T00:38:56.638Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 278.810744ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 954 | 2024-12-19T00:38:56.689Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 295.053358ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 955 | 2024-12-19T00:38:56.821Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 476.792795ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 956 | 2024-12-19T00:38:56.970Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 373.941223ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 957 | 2024-12-19T00:38:57.058Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 483.428865ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 958 | 2024-12-19T00:38:57.147Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 942.633836ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 959 | 2024-12-19T00:38:57.199Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 988.229996ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 960 | 2024-12-19T00:38:57.358Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 976.056875ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 961 | 2024-12-19T00:38:57.417Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 547.44845ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 962 | 2024-12-19T00:38:57.628Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 1.109788841s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 963 | 2024-12-19T00:38:57.997Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 1.272722635s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 964 | 2024-12-19T00:38:58.124Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 688.182848ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 965 | 2024-12-19T00:38:58.223Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 621.141615ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 966 | 2024-12-19T00:38:58.463Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 596.635072ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 967 | 2024-12-19T00:38:58.753Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 1.118083952s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 968 | 2024-12-19T00:38:58.824Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 1.389174584s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 969 | 2024-12-19T00:38:58.855Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 815.896591ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 970 | 2024-12-19T00:38:59.073Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 1.341340994s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 971 | 2024-12-19T00:38:59.285Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 1.290770487s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 972 | 2024-12-19T00:38:59.684Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 1.411496344s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 973 | 2024-12-19T00:38:59.885Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 951.78852ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 974 | 2024-12-19T00:39:00.226Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 1.32034073s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 975 | 2024-12-19T00:39:00.428Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 701.567968ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 976 | 2024-12-19T00:39:00.589Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 1.281077291s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 977 | 2024-12-19T00:39:00.848Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 523.23583ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 978 | 2024-12-19T00:39:01.108Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 766.008185ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 979 | 2024-12-19T00:39:01.139Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 987.838072ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 980 | 2024-12-19T00:39:01.381Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 1.118174487s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 981 | 2024-12-19T00:39:01.558Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 724.398001ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 982 | 2024-12-19T00:39:01.884Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 639.230909ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 983 | 2024-12-19T00:39:01.891Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 875.503604ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 984 | 2024-12-19T00:39:02.139Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 675.809972ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 985 | 2024-12-19T00:39:02.294Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 872.141614ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 986 | 2024-12-19T00:39:02.513Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 562.821061ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 987 | 2024-12-19T00:39:02.534Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 591.00212ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 988 | 2024-12-19T00:39:02.778Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 815.058507ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 989 | 2024-12-19T00:39:02.825Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 614.263022ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 990 | 2024-12-19T00:39:03.086Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 657.578926ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 991 | 2024-12-19T00:39:03.136Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 832.009002ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 992 | 2024-12-19T00:39:03.177Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 809.203805ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 993 | 2024-12-19T00:39:03.450Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 1.150389777s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 994 | 2024-12-19T00:39:03.605Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 986.288269ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 995 | 2024-12-19T00:39:03.755Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 703.520231ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 996 | 2024-12-19T00:39:03.977Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 682.89599ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 997 | 2024-12-19T00:39:03.997Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 945.60907ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 998 | 2024-12-19T00:39:04.469Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 1.317047983s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 999 | 2024-12-19T00:39:04.602Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 688.912846ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 1000 | 2024-12-19T00:39:04.611Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 1.113345565s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 1001 | 2024-12-19T00:39:04.671Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 854.597119ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 1002 | 2024-12-19T00:39:04.953Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 780.956806ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 1003 | 2024-12-19T00:39:05.302Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 1.297095538s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 1004 | 2024-12-19T00:39:05.538Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 1.258817908s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 1005 | 2024-12-19T00:39:05.736Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 1.34151972s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 1006 | 2024-12-19T00:39:05.745Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 665.470468ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 1007 | 2024-12-19T00:39:05.798Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 981.782405ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 1008 | 2024-12-19T00:39:06.421Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 1.282156565s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 1009 | 2024-12-19T00:39:06.611Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 972.986663ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 1010 | 2024-12-19T00:39:06.791Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 1.224176343s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 1011 | 2024-12-19T00:39:06.807Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 1.491980415s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 1012 | 2024-12-19T00:39:07.090Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 1.473816005s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 1013 | 2024-12-19T00:39:07.596Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 1.042014751s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 1014 | 2024-12-19T00:39:07.715Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 546.7725ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 1015 | 2024-12-19T00:39:08.028Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 715.305773ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 1016 | 2024-12-19T00:39:08.248Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 820.405938ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:420
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 1017 | 2024-12-19T00:39:08.272Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 712.023417ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 1018 | 2024-12-19T00:39:08.312Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 975.185276ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 1019 | 2024-12-19T00:39:08.576Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 663.809571ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 1020 | 2024-12-19T00:39:08.650Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 1.21030963s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 1021 | 2024-12-19T00:39:08.753Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 1.242565687s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 1022 | 2024-12-19T00:39:08.997Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 1.06121985s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 1023 | 2024-12-19T00:39:09.253Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 579.694857ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 1024 | 2024-12-19T00:39:09.300Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 641.230388ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 1025 | 2024-12-19T00:39:09.845Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 1.135381651s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 1026 | 2024-12-19T00:39:09.875Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 1.265749349s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 1027 | 2024-12-19T00:39:09.955Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 1.159473841s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 1028 | 2024-12-19T00:39:10.011Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 945.728689ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 1029 | 2024-12-19T00:39:10.073Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 1.442830229s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 1030 | 2024-12-19T00:39:10.466Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:62032
|
| 1031 | 2024-12-19T00:39:10.466Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 29
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:62032
req_id = 2c3e2d0a-14f8-409c-9532-51bd9f1eac1a
response_code = 200
uri = /baseboard
|
| 1032 | 2024-12-19T00:39:10.965Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 1.252888229s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 1033 | 2024-12-19T00:39:10.991Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 655.393171ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 1034 | 2024-12-19T00:39:11.122Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 1.209334383s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 1035 | 2024-12-19T00:39:11.149Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 1.353265262s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 1036 | 2024-12-19T00:39:11.524Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 1.184563641s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 1037 | 2024-12-19T00:39:11.654Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 609.452867ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 1038 | 2024-12-19T00:39:12.228Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 530.820728ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 1039 | 2024-12-19T00:39:12.271Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 916.014656ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 1040 | 2024-12-19T00:39:12.340Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba") failed: Property not found. retry in 1.375936526s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 1041 | 2024-12-19T00:39:12.512Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c") failed: Property not found. retry in 1.20567022s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 1042 | 2024-12-19T00:39:12.718Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf") failed: Property not found. retry in 979.798377ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 1043 | 2024-12-19T00:39:12.767Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7") failed: Property not found. retry in 1.151425096s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 1044 | 2024-12-19T00:39:13.197Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7") failed: Property not found. retry in 1.279627159s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 1045 | 2024-12-19T00:39:14.498Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3310
zone = oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf
|
| 1046 | 2024-12-19T00:39:14.498Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3310
zone = oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba
|
| 1047 | 2024-12-19T00:39:14.498Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3310
zone = oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7
|
| 1048 | 2024-12-19T00:39:14.498Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3310
zone = oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c
|
| 1049 | 2024-12-19T00:39:14.498Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:3310
zone = oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7
|
| 1050 | 2024-12-19T00:39:14.498Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 1051 | 2024-12-19T00:39:14.499Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:199
|
| 1052 | 2024-12-19T00:39:14.499Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 21947362
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:64045
req_id = 41b451dd-f63b-476f-bd83-ac34a01d843e
response_code = 204
uri = /omicron-zones
|
| 1053 | 2024-12-19T00:39:14.533Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:51607
|
| 1054 | 2024-12-19T00:39:14.560Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 26491
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = eb19acf1-f036-4108-a5b0-d37de03e7e3c
response_code = 500
uri = /cockroachdb
|
| 1055 | 2024-12-19T00:39:14.560Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "eb19acf1-f036-4108-a5b0-d37de03e7e3c", "content-length": "124", "date": "Thu, 19 Dec 2024 00:39:14 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "eb19acf1-f036-4108-a5b0-d37de03e7e3c" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 88.170036ms
|
| 1056 | 2024-12-19T00:39:14.676Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 26162
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = c21c27ee-ad3f-4925-ac5e-b0ea9abdf6ef
response_code = 500
uri = /cockroachdb
|
| 1057 | 2024-12-19T00:39:14.676Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "c21c27ee-ad3f-4925-ac5e-b0ea9abdf6ef", "content-length": "124", "date": "Thu, 19 Dec 2024 00:39:14 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "c21c27ee-ad3f-4925-ac5e-b0ea9abdf6ef" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 141.967125ms
|
| 1058 | 2024-12-19T00:39:14.846Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 27163
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = 23dcef54-e5bc-4948-a86e-a95cd0dcbc18
response_code = 500
uri = /cockroachdb
|
| 1059 | 2024-12-19T00:39:14.846Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "23dcef54-e5bc-4948-a86e-a95cd0dcbc18", "content-length": "124", "date": "Thu, 19 Dec 2024 00:39:14 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "23dcef54-e5bc-4948-a86e-a95cd0dcbc18" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 207.374106ms
|
| 1060 | 2024-12-19T00:39:15.088Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1061 | 2024-12-19T00:39:35.396Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1062 | 2024-12-19T00:39:38.964Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 23909089
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = 03cd9689-0398-4e92-882c-4c12b6f02068
response_code = 500
uri = /cockroachdb
|
| 1063 | 2024-12-19T00:39:38.971Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "03cd9689-0398-4e92-882c-4c12b6f02068", "content-length": "124", "date": "Thu, 19 Dec 2024 00:39:38 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "03cd9689-0398-4e92-882c-4c12b6f02068" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 132.725747ms
|
| 1064 | 2024-12-19T00:39:38.988Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 2.98968073s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:420
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 1065 | 2024-12-19T00:39:39.143Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1066 | 2024-12-19T00:39:39.255Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1067 | 2024-12-19T00:39:44.340Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 5235314
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = a8175414-f6bf-4476-9fc9-698aafd2aba1
response_code = 500
uri = /cockroachdb
|
| 1068 | 2024-12-19T00:39:44.341Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56565
|
| 1069 | 2024-12-19T00:39:44.341Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a8175414-f6bf-4476-9fc9-698aafd2aba1", "content-length": "124", "date": "Thu, 19 Dec 2024 00:39:44 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a8175414-f6bf-4476-9fc9-698aafd2aba1" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 189.508529ms
|
| 1070 | 2024-12-19T00:39:44.342Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 525
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56565
req_id = dde95b1d-132c-46c0-b8fa-84e2b6656d0a
response_code = 200
uri = /baseboard
|
| 1071 | 2024-12-19T00:39:44.560Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1072 | 2024-12-19T00:39:44.676Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1073 | 2024-12-19T00:39:50.031Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 5497953
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = 54a7474e-9d13-4004-ab6c-5fa8fc5d345c
response_code = 500
uri = /cockroachdb
|
| 1074 | 2024-12-19T00:39:50.032Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "54a7474e-9d13-4004-ab6c-5fa8fc5d345c", "content-length": "124", "date": "Thu, 19 Dec 2024 00:39:50 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "54a7474e-9d13-4004-ab6c-5fa8fc5d345c" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 144.726207ms
|
| 1075 | 2024-12-19T00:39:50.206Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1076 | 2024-12-19T00:39:50.321Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1077 | 2024-12-19T00:40:03.435Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 13257329
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = b36e06ae-59db-45eb-9d1b-594c73d81c6f
response_code = 500
uri = /cockroachdb
|
| 1078 | 2024-12-19T00:40:03.435Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b36e06ae-59db-45eb-9d1b-594c73d81c6f", "content-length": "124", "date": "Thu, 19 Dec 2024 00:40:03 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b36e06ae-59db-45eb-9d1b-594c73d81c6f" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 446.74437ms
|
| 1079 | 2024-12-19T00:40:03.439Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 4.511325458s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:420
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 1080 | 2024-12-19T00:40:03.911Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1081 | 2024-12-19T00:40:04.024Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1082 | 2024-12-19T00:40:12.373Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 8489711
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = 2f6945aa-f538-4dc3-8d55-bcee2b283c01
response_code = 500
uri = /cockroachdb
|
| 1083 | 2024-12-19T00:40:12.374Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "2f6945aa-f538-4dc3-8d55-bcee2b283c01", "content-length": "124", "date": "Thu, 19 Dec 2024 00:40:12 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "2f6945aa-f538-4dc3-8d55-bcee2b283c01" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 484.485098ms
|
| 1084 | 2024-12-19T00:40:12.889Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1085 | 2024-12-19T00:40:13.008Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1086 | 2024-12-19T00:40:19.838Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 6977897
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = 6b3d6ee2-615d-4903-983d-f8d4e814b182
response_code = 500
uri = /cockroachdb
|
| 1087 | 2024-12-19T00:40:19.838Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:64367
|
| 1088 | 2024-12-19T00:40:19.839Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6b3d6ee2-615d-4903-983d-f8d4e814b182", "content-length": "124", "date": "Thu, 19 Dec 2024 00:40:19 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6b3d6ee2-615d-4903-983d-f8d4e814b182" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 457.030831ms
|
| 1089 | 2024-12-19T00:40:19.839Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 68
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:64367
req_id = 7714bf71-24c6-4d27-a7ff-1ceba4177d62
response_code = 200
uri = /baseboard
|
| 1090 | 2024-12-19T00:40:20.326Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1091 | 2024-12-19T00:40:20.439Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1092 | 2024-12-19T00:40:27.417Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 7119595
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = 6ff41acc-9fc6-4b9d-a095-c1666a1d944a
response_code = 500
uri = /cockroachdb
|
| 1093 | 2024-12-19T00:40:27.417Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6ff41acc-9fc6-4b9d-a095-c1666a1d944a", "content-length": "124", "date": "Thu, 19 Dec 2024 00:40:27 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6ff41acc-9fc6-4b9d-a095-c1666a1d944a" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 471.830844ms
|
| 1094 | 2024-12-19T00:40:27.420Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 6.243760831s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:420
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 1095 | 2024-12-19T00:40:27.917Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1096 | 2024-12-19T00:40:28.031Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1097 | 2024-12-19T00:40:37.671Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 9780972
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = e7331efc-e739-4d11-8fe7-b7164e78578c
response_code = 500
uri = /cockroachdb
|
| 1098 | 2024-12-19T00:40:37.671Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e7331efc-e739-4d11-8fe7-b7164e78578c", "content-length": "124", "date": "Thu, 19 Dec 2024 00:40:37 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e7331efc-e739-4d11-8fe7-b7164e78578c" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 311.272887ms
|
| 1099 | 2024-12-19T00:40:38.012Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1100 | 2024-12-19T00:40:38.121Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1101 | 2024-12-19T00:40:46.758Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 8774657
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = c9833179-2a4f-4e3f-b4f6-1f57c67aa4a5
response_code = 500
uri = /cockroachdb
|
| 1102 | 2024-12-19T00:40:46.759Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "c9833179-2a4f-4e3f-b4f6-1f57c67aa4a5", "content-length": "124", "date": "Thu, 19 Dec 2024 00:40:46 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "c9833179-2a4f-4e3f-b4f6-1f57c67aa4a5" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 683.237759ms
|
| 1103 | 2024-12-19T00:40:47.469Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1104 | 2024-12-19T00:40:47.585Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1105 | 2024-12-19T00:41:05.069Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 17627187
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = a252a69d-54fb-48b8-82a6-dc53cdc1645b
response_code = 500
uri = /cockroachdb
|
| 1106 | 2024-12-19T00:41:05.070Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:50224
|
| 1107 | 2024-12-19T00:41:05.070Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a252a69d-54fb-48b8-82a6-dc53cdc1645b", "content-length": "124", "date": "Thu, 19 Dec 2024 00:41:05 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a252a69d-54fb-48b8-82a6-dc53cdc1645b" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 1.050340561s
|
| 1108 | 2024-12-19T00:41:05.073Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 17.25744959s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:420
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 1109 | 2024-12-19T00:41:06.149Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1110 | 2024-12-19T00:41:07.307Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1111 | 2024-12-19T00:41:15.487Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 9366119
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = a2579291-416d-426c-8e00-752edc559605
response_code = 500
uri = /cockroachdb
|
| 1112 | 2024-12-19T00:41:15.488Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a2579291-416d-426c-8e00-752edc559605", "content-length": "124", "date": "Thu, 19 Dec 2024 00:41:15 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a2579291-416d-426c-8e00-752edc559605" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 1.459322889s
|
| 1113 | 2024-12-19T00:41:16.976Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1114 | 2024-12-19T00:41:17.085Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1115 | 2024-12-19T00:41:32.250Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 15301104
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = 47415430-492f-4f61-a0c0-578376e972ca
response_code = 500
uri = /cockroachdb
|
| 1116 | 2024-12-19T00:41:32.250Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "47415430-492f-4f61-a0c0-578376e972ca", "content-length": "124", "date": "Thu, 19 Dec 2024 00:41:32 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "47415430-492f-4f61-a0c0-578376e972ca" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 678.857632ms
|
| 1117 | 2024-12-19T00:41:32.960Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1118 | 2024-12-19T00:41:33.090Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1119 | 2024-12-19T00:41:40.585Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 7654835
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = d7ec36b6-84c5-4a2a-8e0e-5d1d8ecae5af
response_code = 500
uri = /cockroachdb
|
| 1120 | 2024-12-19T00:41:40.585Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:55055
|
| 1121 | 2024-12-19T00:41:40.585Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d7ec36b6-84c5-4a2a-8e0e-5d1d8ecae5af", "content-length": "124", "date": "Thu, 19 Dec 2024 00:41:40 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d7ec36b6-84c5-4a2a-8e0e-5d1d8ecae5af" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 909.606614ms
|
| 1122 | 2024-12-19T00:41:40.586Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 104
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:55055
req_id = 9197dae0-3f48-4b40-895a-56684a6c05d6
response_code = 200
uri = /baseboard
|
| 1123 | 2024-12-19T00:41:41.526Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1124 | 2024-12-19T00:41:41.643Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1125 | 2024-12-19T00:42:05.463Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 23966254
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = a5794071-cc7b-4624-ba24-2c42f4315b5d
response_code = 500
uri = /cockroachdb
|
| 1126 | 2024-12-19T00:42:05.464Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a5794071-cc7b-4624-ba24-2c42f4315b5d", "content-length": "124", "date": "Thu, 19 Dec 2024 00:42:05 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a5794071-cc7b-4624-ba24-2c42f4315b5d" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 1.598898035s
|
| 1127 | 2024-12-19T00:42:05.468Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 38.625099715s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:420
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 1128 | 2024-12-19T00:42:07.092Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1129 | 2024-12-19T00:42:07.204Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1130 | 2024-12-19T00:42:15.676Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 8612363
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = 3e77e2d6-cf48-4d7c-af80-2c51bb614833
response_code = 500
uri = /cockroachdb
|
| 1131 | 2024-12-19T00:42:15.677Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:36300
|
| 1132 | 2024-12-19T00:42:15.677Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3e77e2d6-cf48-4d7c-af80-2c51bb614833", "content-length": "124", "date": "Thu, 19 Dec 2024 00:42:15 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3e77e2d6-cf48-4d7c-af80-2c51bb614833" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 2.044919738s
|
| 1133 | 2024-12-19T00:42:15.677Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 102
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:36300
req_id = e7d2f2b3-60e5-480b-a388-12123cc0e17f
response_code = 200
uri = /baseboard
|
| 1134 | 2024-12-19T00:42:17.751Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1135 | 2024-12-19T00:42:17.866Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1136 | 2024-12-19T00:42:25.048Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 7325053
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = 16a8e5f7-1373-422e-8bce-59c9298433c6
response_code = 500
uri = /cockroachdb
|
| 1137 | 2024-12-19T00:42:25.049Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "16a8e5f7-1373-422e-8bce-59c9298433c6", "content-length": "124", "date": "Thu, 19 Dec 2024 00:42:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "16a8e5f7-1373-422e-8bce-59c9298433c6" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 2.159785639s
|
| 1138 | 2024-12-19T00:42:27.237Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1139 | 2024-12-19T00:42:27.348Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1140 | 2024-12-19T00:42:34.495Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 7286049
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = 60ae877a-f862-4e0c-bc25-42cb280673af
response_code = 500
uri = /cockroachdb
|
| 1141 | 2024-12-19T00:42:34.496Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "60ae877a-f862-4e0c-bc25-42cb280673af", "content-length": "124", "date": "Thu, 19 Dec 2024 00:42:34 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "60ae877a-f862-4e0c-bc25-42cb280673af" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 4.219573675s
|
| 1142 | 2024-12-19T00:42:38.744Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1143 | 2024-12-19T00:42:38.857Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1144 | 2024-12-19T00:42:46.319Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 7602942
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = 78b8cdd5-4c84-45da-a39e-14241e42e368
response_code = 500
uri = /cockroachdb
|
| 1145 | 2024-12-19T00:42:46.320Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:64561
|
| 1146 | 2024-12-19T00:42:46.320Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "78b8cdd5-4c84-45da-a39e-14241e42e368", "content-length": "124", "date": "Thu, 19 Dec 2024 00:42:46 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "78b8cdd5-4c84-45da-a39e-14241e42e368" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 5.19925374s
|
| 1147 | 2024-12-19T00:42:46.322Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 95
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:64561
req_id = d786d620-a19b-4e21-8881-c9994a35a99a
response_code = 200
uri = /baseboard
|
| 1148 | 2024-12-19T00:42:51.547Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1149 | 2024-12-19T00:42:51.658Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1150 | 2024-12-19T00:43:16.092Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 24572100
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = bcf7fc03-452d-4ab4-97b6-f00dd29c6f0f
response_code = 500
uri = /cockroachdb
|
| 1151 | 2024-12-19T00:43:16.093Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "bcf7fc03-452d-4ab4-97b6-f00dd29c6f0f", "content-length": "124", "date": "Thu, 19 Dec 2024 00:43:16 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "bcf7fc03-452d-4ab4-97b6-f00dd29c6f0f" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 2.406606897s
|
| 1152 | 2024-12-19T00:43:16.094Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:862
|
| 1153 | 2024-12-19T00:43:16.096Z | WARN | SledAgent (producer-registration-task): failed to register as a producer with Nexus, will retry
delay = 59.283718609s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)"
file = oximeter/producer/src/lib.rs:420
sled_id = e358e662-7828-4f7f-85a4-21e089483ee6
|
| 1154 | 2024-12-19T00:43:16.355Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:47701
|
| 1155 | 2024-12-19T00:43:16.356Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 43
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:47701
req_id = 88231df0-a07f-4db0-a0a6-6587d0694545
response_code = 200
uri = /baseboard
|
| 1156 | 2024-12-19T00:43:18.527Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1157 | 2024-12-19T00:43:18.640Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1158 | 2024-12-19T00:43:25.749Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 7249185
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = 03a00770-2a3a-4098-85a6-29f10216bb04
response_code = 500
uri = /cockroachdb
|
| 1159 | 2024-12-19T00:43:25.750Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "03a00770-2a3a-4098-85a6-29f10216bb04", "content-length": "124", "date": "Thu, 19 Dec 2024 00:43:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "03a00770-2a3a-4098-85a6-29f10216bb04" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 5.535295984s
|
| 1160 | 2024-12-19T00:43:31.313Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1161 | 2024-12-19T00:43:31.420Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1162 | 2024-12-19T00:43:48.707Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 17420363
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = 285e3053-38a8-4559-821e-e411f4598815
response_code = 500
uri = /cockroachdb
|
| 1163 | 2024-12-19T00:43:48.707Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45801
|
| 1164 | 2024-12-19T00:43:48.707Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "285e3053-38a8-4559-821e-e411f4598815", "content-length": "124", "date": "Thu, 19 Dec 2024 00:43:48 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "285e3053-38a8-4559-821e-e411f4598815" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 3.527113508s
|
| 1165 | 2024-12-19T00:43:48.707Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863
latency_us = 80
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45801
req_id = 4958953c-666d-4b7c-a489-24f0d43b4d63
response_code = 200
uri = /baseboard
|
| 1166 | 2024-12-19T00:43:52.266Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221
file = sled-agent/src/services.rs:3743
|
| 1167 | 2024-12-19T00:43:52.383Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3761
|
| 1168 | 2024-12-19T00:43:59.971Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near ")": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running "sql"\n
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851
latency_us = 7385641
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:51607
req_id = 3a6c48d7-6611-4013-8491-142f005c938a
response_code = 500
uri = /cockroachdb
|
| 1169 | 2024-12-19T00:43:59.971Z | WARN | SledAgent (RSS): Failed to initialize CockroachDB
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3a6c48d7-6611-4013-8491-142f005c938a", "content-length": "124", "date": "Thu, 19 Dec 2024 00:43:59 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3a6c48d7-6611-4013-8491-142f005c938a" }
file = sled-agent/src/rack_setup/service.rs:1157
retry_after = 7.394290669s
|