[ Dec 19 00:37:47 Enabled. ] [ Dec 19 00:37:47 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Dec 19 00:37:47 Method "start" exited with status 0. ] [ Dec 19 00:37:47 Rereading configuration. ] [ Dec 19 00:37:47 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:47.955910129Z","hostname":"sock","pid":1566,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:47.956061261Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.008594835Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.008627996Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.123081277Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.132165194Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.140397294Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.745491135Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.886724498Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.88680951Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.886864221Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.886955503Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.886971374Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.886981524Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.890015466Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:48.890022356Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.890038697Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.890050137Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.890080707Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.890137709Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.890240401Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.890259261Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.890274022Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.902389752Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:48.902407302Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:49.001238Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:49.460219898Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:49.460258159Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:49.460264399Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:49.460277539Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:49.460286109Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:49.4603189Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:49.46032366Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:49.46032948Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:49.46033356Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:49.460366561Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:49.469630852Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:49.469650802Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:49.557359922Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032339189Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.03238407Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.03240354Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.03241284Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032420901Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.03242577Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032442681Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032465891Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032485352Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032488952Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032501482Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032508042Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032550693Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032567053Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032570664Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032578204Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032593284Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032599464Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032604274Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032611044Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032614224Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032620545Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032638055Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032642625Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032645875Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032653105Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032671106Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032675056Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032678986Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032685056Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032688946Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032692586Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032716716Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032721527Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032727277Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032730237Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032734627Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032737637Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032759437Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032764697Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032768677Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032772208Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032790878Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032795238Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032801248Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032805078Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032834559Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032839329Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032842689Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032847569Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032851379Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032855539Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.03288177Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.03288643Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.0328898Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.03289985Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.03290352Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032930021Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032939031Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032942551Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032946021Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.032996322Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.033009933Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.033021513Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.033038123Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.033042853Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.033046793Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.033078074Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.033086294Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.043608041Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:736"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.043640842Z","hostname":"sock","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999951209s","next_cleanup":"Instant { tv_sec: 1394, tv_nsec: 796520085 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.043679643Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3947"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.043688083Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.043740844Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.046621643Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.047019122Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.048677396Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: Some(\n None,\n ),\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.048918081Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1203"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.048924981Z","hostname":"sock","pid":1566,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.04935131Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.04937532Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.049424541Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.049456302Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.049468412Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.049476502Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS not previously run. Creating plans.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.049481172Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1252"} {"msg":"Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.049491973Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.049727877Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.04984499Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1284"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.04985019Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:66"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.04985373Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:39"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.049889571Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1316"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.049939772Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.049944842Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.050108485Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.050267289Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.050369751Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: e358e662-7828-4f7f-85a4-21e089483ee6 (sled), rack_id: ec268bea-b1b9-4b27-9476-6ec4bc3257c8, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.050755459Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4244"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.05080868Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4644"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.05520624Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.061261705Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.061372228Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065455292Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065522043Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065649336Z","hostname":"sock","pid":1566,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:42770","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: e358e662-7828-4f7f-85a4-21e089483ee6 (sled), rack_id: ec268bea-b1b9-4b27-9476-6ec4bc3257c8, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065788049Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065802169Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065874171Z","hostname":"sock","pid":1566,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065887441Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065897891Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:809"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065902051Z","hostname":"sock","pid":1566,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065910251Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065917331Z","hostname":"sock","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065927601Z","hostname":"sock","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065938662Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:798"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065945172Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:826"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.065949522Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:592"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.066030924Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.066042794Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:697","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.080905681Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.101336202Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.118412474Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.129126555Z","hostname":"sock","pid":1566,"sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:415"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.129157896Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.129175956Z","hostname":"sock","pid":1566,"sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:427"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.129231907Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 101.304741ms)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.132500665Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"05ee4050-057d-4f62-b7bb-420f3da0890f\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:37:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"05ee4050-057d-4f62-b7bb-420f3da0890f\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_484faba5-5bf0-4a1e-b697-76cf9fed3ca8/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.143754127Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_484faba5-5bf0-4a1e-b697-76cf9fed3ca8/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.166302792Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_484faba5-5bf0-4a1e-b697-76cf9fed3ca8/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.166340183Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.171700193Z","hostname":"sock","pid":1566,"sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:442"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.171716844Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:128"} {"msg":"Ensuring dataset oxi_484faba5-5bf0-4a1e-b697-76cf9fed3ca8/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.171723734Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:138"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.235105641Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 169.216422ms)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.235595011Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6e76b941-3a40-4969-8737-1535b7b0f215\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:37:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6e76b941-3a40-4969-8737-1535b7b0f215\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.257860741Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:173"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.406002166Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 198.707334ms)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.406631949Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6d7a68d1-cb4d-460f-87e3-e3b62413fd2d\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:37:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6d7a68d1-cb4d-460f-87e3-e3b62413fd2d\" }"} {"msg":"Mounting oxi_484faba5-5bf0-4a1e-b697-76cf9fed3ca8/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.476297426Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:181"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.488399666Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.488486648Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.488518398Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.488550989Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:195"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.606630175Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 161.054176ms)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.607313989Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fc5a6e38-fa6e-4686-b9f0-2602c189a13a\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:37:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fc5a6e38-fa6e-4686-b9f0-2602c189a13a\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.769501764Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 160.466284ms)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.770129597Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6d926f56-fd36-4b17-bacb-df6b3dbe3790\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:37:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6d926f56-fd36-4b17-bacb-df6b3dbe3790\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.931316502Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 373.024171ms)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:50.931838083Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"97a79d25-f959-40ee-9426-1deb77a97a5f\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:37:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"97a79d25-f959-40ee-9426-1deb77a97a5f\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:51.224880607Z","hostname":"sock","pid":1566,"sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:51.230389281Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:42252","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:51.230421752Z","hostname":"sock","pid":1566,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:51.230495233Z","hostname":"sock","pid":1566,"file":"oximeter/producer/src/lib.rs:279","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:279","interval":"30s","address":"[fd00:1122:3344:101::1]:42252","producer_id":"e358e662-7828-4f7f-85a4-21e089483ee6"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:51.231746969Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"180.269804ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:51.232656178Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:51.232667348Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:51.232726929Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:920","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:51.232738529Z","hostname":"sock","pid":1566,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:51.305831907Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:51.413567269Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"257.260767ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:51.672030011Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.307208878s"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:52.980940089Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.37311918s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:53.738854243Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2668"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:53.738874033Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2913"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:53.738878694Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2538"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:53.738885554Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2585"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:53.738939235Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2994"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:53.738943825Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3039"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:53.740938546Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2838"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:53.742546859Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2579"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:53.742664161Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:53.742811395Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 63.176158ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:54.024217619Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 141.165816ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:54.235502327Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 198.987817ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:54.422057026Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 450.580995ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:54.769036093Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 886.081334ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:55.362153227Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:55.362924873Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"3.134620657s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 754.348345ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:56.368425683Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 927.064474ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:57.230975885Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.007630281s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:58.370619972Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:58.49994982Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.180234554s"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e358e662-7828-4f7f-85a4-21e089483ee6)","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:37:59.902434009Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:59.932142112Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:59.932592692Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12348","component":"dropshot (Repo Depot)","component":"ArtifactStore","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:59.933869108Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:59.933890218Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:59.933909819Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:59.933919079Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:59.934083442Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:59.934250776Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:59.934295017Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: Some(None), bgp_peers: [], autoneg: false, lldp: None, tx_eq: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:59.934303757Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3947"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:37:59.934356778Z","hostname":"sock","pid":1566,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:42770","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.125776746Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.125850608Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.131646257Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4268","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.131664267Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4284"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.131680008Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.40035938Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:65124"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.400655946Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.400664276Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.400713317Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.400745368Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.40084435Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"cf1baf1f-c252-4e2a-9664-99d270bad7a3","remote_addr":"[fd00:1122:3344:101::1]:65124","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":305,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.431285458Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:48883"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.431417611Z","hostname":"sock","pid":1566,"uri":"/sled-role","method":"GET","req_id":"f0c782f6-ce4b-4da0-91fe-819aefe98a5c","remote_addr":"[fd00:1122:3344:101::1]:48883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":51,"response_code":"200"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 34585501-77f9-46aa-bbe8-ab9978160629 (physical_disk), pool_id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 32944af9-f1bc-460a-8b1f-0f379977d412 (physical_disk), pool_id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 5339a28f-8415-4ebc-975f-7a8352cc53a6 (physical_disk), pool_id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 21cce821-1fd9-4b57-9e1b-77489c0dba86 (physical_disk), pool_id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: d6ad0783-9d78-40d5-9cf3-2c223ca50cbf (physical_disk), pool_id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 3735fb39-3079-43bb-9c04-9ad82213b986 (physical_disk), pool_id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: abda9141-7cf7-44e6-addd-c2b3f5051e7d (physical_disk), pool_id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 2dbaf12a-2a59-4be4-9274-44f1a0e5842e (physical_disk), pool_id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: be669f5b-7fba-4429-9388-191e847a9125 (physical_disk), pool_id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.461222776Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:385"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.461319197Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:45008"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.461671005Z","hostname":"sock","pid":1566,"sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:896"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 34585501-77f9-46aa-bbe8-ab9978160629 (physical_disk), pool_id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 32944af9-f1bc-460a-8b1f-0f379977d412 (physical_disk), pool_id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 5339a28f-8415-4ebc-975f-7a8352cc53a6 (physical_disk), pool_id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 21cce821-1fd9-4b57-9e1b-77489c0dba86 (physical_disk), pool_id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: d6ad0783-9d78-40d5-9cf3-2c223ca50cbf (physical_disk), pool_id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 3735fb39-3079-43bb-9c04-9ad82213b986 (physical_disk), pool_id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: abda9141-7cf7-44e6-addd-c2b3f5051e7d (physical_disk), pool_id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 2dbaf12a-2a59-4be4-9274-44f1a0e5842e (physical_disk), pool_id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: be669f5b-7fba-4429-9388-191e847a9125 (physical_disk), pool_id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.461725876Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.461744216Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.461755166Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.461760227Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1171"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.461790867Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.461796217Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.461802057Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.461806408Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.606954062Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:00.606978762Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"1a77147c-c146-4e56-8066-57b7b2b29250 (zpool)"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:01.052088903Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.265356112Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:01.266957485Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14076285699142498485"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:01.267082638Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15791617063128661124"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.312163738Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.319874787Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.319956918Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.319966349Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.319973899Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.319979539Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.320095031Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.320101732Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.355537203Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.363410155Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_1a77147c-c146-4e56-8066-57b7b2b29250","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.534036224Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.541832685Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.541863886Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.541872686Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.541879466Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.541892196Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.552035746Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.552062266Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.647304121Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.654988399Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.65503828Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.65504967Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.655127102Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.655139762Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.655275235Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.655283445Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.692753468Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.70058723Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.869544755Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.877093971Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.877123301Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.877130782Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.877139042Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.877170112Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.886873443Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.886902073Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.983123517Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.991154623Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.991185734Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.991194824Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.991205284Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.991236275Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.991321037Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:01.991333277Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.026726247Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.035507078Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:02.054345666Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.213006939Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.22079547Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.220840431Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.220848521Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.220854241Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.220860411Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.230871658Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.230894448Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.327015851Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.334780611Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.334816992Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.334823792Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.334882153Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.334893964Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.335024156Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.335032176Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.369376935Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.37690796Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.548690223Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.556918093Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.556957074Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.556991405Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.557001665Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.557009235Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.566310337Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.566335458Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.6624552Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.670452005Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.670518687Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.670528267Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.670544757Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.670555197Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.670723151Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.670738741Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.706299494Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.714063925Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.885193455Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.892891823Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.892932384Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.892939604Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.892946515Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.892989065Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.902987782Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.903018112Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"b0ef393a-1bdf-419f-872d-787b91138b01 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:02.999064583Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.006786963Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.006821263Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.006848194Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.006860774Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.006870874Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.007054028Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.007063208Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.040434817Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.048126185Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:03.059592562Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_b0ef393a-1bdf-419f-872d-787b91138b01","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.224656406Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.232824565Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.232870756Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.232877616Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.232893146Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.232898636Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.242173248Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.242200688Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.335577534Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.343230082Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.343255193Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.343262803Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.343277383Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.343325124Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.343455727Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.343465927Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.379781626Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.38772304Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.562198439Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.570579132Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.570620583Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.570627193Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.570632913Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.570639173Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.582742893Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.582772764Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"94913555-b694-4ecc-9c16-7af8c3b569ae (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.681716264Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.689526735Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.689567916Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.689574796Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.689614017Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.689635438Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.689818792Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.689829492Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:03.693513108Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"13.775685162s"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.725677781Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.733279608Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_94913555-b694-4ecc-9c16-7af8c3b569ae","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.909650316Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.917540499Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.91757871Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.91758695Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.91759405Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.917632931Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.928568666Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:03.928604857Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.027027507Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.036209726Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.036245967Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.036251578Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.036293278Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.036302508Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.036434431Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.036458202Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:04.061547759Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.073569467Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.08146726Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_d634c5ac-cef9-463f-93e8-f083982a2a97","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.255866308Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.263578736Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.263617637Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.263690249Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.263821781Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(2), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 484faba5-5bf0-4a1e-b697-76cf9fed3ca8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 88c34544-1e3c-4ff1-a021-704e84e5eabd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.263883653Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.263891653Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.264049806Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.264289381Z","hostname":"sock","pid":1566,"sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:900"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.264305811Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.264316002Z","hostname":"sock","pid":1566,"sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:926","generation":"Generation(2)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.264320712Z","hostname":"sock","pid":1566,"sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:931"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.264324422Z","hostname":"sock","pid":1566,"sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:936"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.264350003Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:122","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.264359343Z","hostname":"sock","pid":1566,"sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:942"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.264394453Z","hostname":"sock","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.264399664Z","hostname":"sock","pid":1566,"sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:947"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.264464735Z","hostname":"sock","pid":1566,"uri":"/omicron-physical-disks","method":"PUT","req_id":"15bb1bdd-5791-4ce2-9733-f93df7355451","remote_addr":"[fd00:1122:3344:101::1]:45008","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3803010,"response_code":"200"} {"msg":"attempting to set up sled's datasets: DatasetsConfig { generation: Generation(1), datasets: {02bdb46f-1f7a-4495-8e8a-a1c8f0a03bc0 (dataset): DatasetConfig { id: 02bdb46f-1f7a-4495-8e8a-a1c8f0a03bc0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 08299bfb-0800-4ca8-9b8d-9b576960e166 (dataset): DatasetConfig { id: 08299bfb-0800-4ca8-9b8d-9b576960e166 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 09cc2927-2cf5-4413-8be0-f1ddbc79984d (dataset): DatasetConfig { id: 09cc2927-2cf5-4413-8be0-f1ddbc79984d (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 10dda82e-9265-49e8-b8f3-2a4a9c4c164b (dataset): DatasetConfig { id: 10dda82e-9265-49e8-b8f3-2a4a9c4c164b (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_7acb7f99-08f6-422b-b4ef-5456a5a44c23\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 125f36d7-301e-406a-bc11-db67930c216a (dataset): DatasetConfig { id: 125f36d7-301e-406a-bc11-db67930c216a (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 1356fa90-9862-4602-8bec-987f6eefba47 (dataset): DatasetConfig { id: 1356fa90-9862-4602-8bec-987f6eefba47 (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1542d45f-d5fe-49f7-aac2-c5dc5b736508 (dataset): DatasetConfig { id: 1542d45f-d5fe-49f7-aac2-c5dc5b736508 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1ba3c1dd-2484-4777-8624-adf9adabad51 (dataset): DatasetConfig { id: 1ba3c1dd-2484-4777-8624-adf9adabad51 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_70eb21d3-150c-4d21-bcc3-bbf3bd06c293\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1bd3a984-3b13-48a9-9fd9-5a85e418f46b (dataset): DatasetConfig { id: 1bd3a984-3b13-48a9-9fd9-5a85e418f46b (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2385dbc1-3195-45e0-8662-43c4a515014c (dataset): DatasetConfig { id: 2385dbc1-3195-45e0-8662-43c4a515014c (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 25705b62-3c7a-45bd-9cba-99bd5767da59 (dataset): DatasetConfig { id: 25705b62-3c7a-45bd-9cba-99bd5767da59 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_d5087ea8-b098-443c-b9ed-af8abfbf27e8\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 26790d95-ebdd-44fd-86e5-9dd4ba3e4f21 (dataset): DatasetConfig { id: 26790d95-ebdd-44fd-86e5-9dd4ba3e4f21 (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 26e0b6b0-ce70-43ca-980b-0cf12a292bcf (dataset): DatasetConfig { id: 26e0b6b0-ce70-43ca-980b-0cf12a292bcf (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 2d1e47ac-ebac-444a-b49e-9d3573199828 (dataset): DatasetConfig { id: 2d1e47ac-ebac-444a-b49e-9d3573199828 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 31e811be-26c1-4ef9-b32e-05600feef342 (dataset): DatasetConfig { id: 31e811be-26c1-4ef9-b32e-05600feef342 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_33c09be7-40a9-46e5-829a-6af4b245eb8d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 338b98ec-77e1-415d-8738-e1eea139be02 (dataset): DatasetConfig { id: 338b98ec-77e1-415d-8738-e1eea139be02 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_6bb5054e-2e33-47c5-8591-838209d609a0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3890ce90-eb11-4f7f-b73f-41bc8c3549cd (dataset): DatasetConfig { id: 3890ce90-eb11-4f7f-b73f-41bc8c3549cd (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 3a270f45-f343-4a4c-a795-e1d1d4026117 (dataset): DatasetConfig { id: 3a270f45-f343-4a4c-a795-e1d1d4026117 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a605806-3dd8-48f6-a271-4921a40cab01 (dataset): DatasetConfig { id: 3a605806-3dd8-48f6-a271-4921a40cab01 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4801adcb-51da-4c43-a6b8-74c9ab7d469e (dataset): DatasetConfig { id: 4801adcb-51da-4c43-a6b8-74c9ab7d469e (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 53e73ca6-22a3-4760-8d8c-13ba65fd0138 (dataset): DatasetConfig { id: 53e73ca6-22a3-4760-8d8c-13ba65fd0138 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 57924c57-7a89-42cf-b574-6af7898fbf20 (dataset): DatasetConfig { id: 57924c57-7a89-42cf-b574-6af7898fbf20 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 57d1f209-4bbf-4b28-8fc6-71af4aa89766 (dataset): DatasetConfig { id: 57d1f209-4bbf-4b28-8fc6-71af4aa89766 (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 57d5be5f-b1fb-493e-abd9-49787f94f8bd (dataset): DatasetConfig { id: 57d5be5f-b1fb-493e-abd9-49787f94f8bd (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_6a23c2a5-006e-4c29-a54c-390deecea882\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 58434d8e-917a-40ea-a69c-83af306968fe (dataset): DatasetConfig { id: 58434d8e-917a-40ea-a69c-83af306968fe (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_60daa419-d06f-44da-8437-507d9480bdc5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5b27c472-485e-4e9d-9aba-7cd17c6bd10d (dataset): DatasetConfig { id: 5b27c472-485e-4e9d-9aba-7cd17c6bd10d (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5e7c3ef9-e5f3-4f5b-afd5-97a14d72a56c (dataset): DatasetConfig { id: 5e7c3ef9-e5f3-4f5b-afd5-97a14d72a56c (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 626f8ac1-7e2f-4555-a521-96f6d9d1c49f (dataset): DatasetConfig { id: 626f8ac1-7e2f-4555-a521-96f6d9d1c49f (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 628dbe7b-bcea-4ffc-9f4e-c56ad9549e5e (dataset): DatasetConfig { id: 628dbe7b-bcea-4ffc-9f4e-c56ad9549e5e (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 63082ad4-1646-44cb-a619-f99202ef3e1d (dataset): DatasetConfig { id: 63082ad4-1646-44cb-a619-f99202ef3e1d (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_2e431855-2d60-4eca-bbca-519c0a7d8124\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6cf98f37-21a1-4ff0-b9a6-3d0efc3ee8be (dataset): DatasetConfig { id: 6cf98f37-21a1-4ff0-b9a6-3d0efc3ee8be (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_1bfa1410-b334-4a2b-ac4e-77fa9c04ad35\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6ee13928-1186-4e68-ae12-f5f0880b9bda (dataset): DatasetConfig { id: 6ee13928-1186-4e68-ae12-f5f0880b9bda (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 749db2ac-f07c-459b-8dfe-2e14e25a342c (dataset): DatasetConfig { id: 749db2ac-f07c-459b-8dfe-2e14e25a342c (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 758f88cf-7734-4193-a836-12f01f4bbff8 (dataset): DatasetConfig { id: 758f88cf-7734-4193-a836-12f01f4bbff8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_51618710-2d9e-4911-bfc3-577586e96ca3\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 787d82fb-5c78-42bb-8d95-7bd9d94d3c37 (dataset): DatasetConfig { id: 787d82fb-5c78-42bb-8d95-7bd9d94d3c37 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7fd0b8db-fc34-4652-b220-93eaa7beee17 (dataset): DatasetConfig { id: 7fd0b8db-fc34-4652-b220-93eaa7beee17 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8948f738-5705-4437-8259-8bb3c045db11 (dataset): DatasetConfig { id: 8948f738-5705-4437-8259-8bb3c045db11 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8e32d5e5-5fbe-40f4-87a8-39a7939aa467 (dataset): DatasetConfig { id: 8e32d5e5-5fbe-40f4-87a8-39a7939aa467 (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9237f071-5643-491d-9e34-1f622d5fe205 (dataset): DatasetConfig { id: 9237f071-5643-491d-9e34-1f622d5fe205 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 93bcddc6-63cc-4c1b-a60f-47b0fb416c23 (dataset): DatasetConfig { id: 93bcddc6-63cc-4c1b-a60f-47b0fb416c23 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 99c77c44-c34f-4aa4-9f16-3e4edb45d1ec (dataset): DatasetConfig { id: 99c77c44-c34f-4aa4-9f16-3e4edb45d1ec (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9aa0d225-1893-4fc0-8d5d-40cf890c4067 (dataset): DatasetConfig { id: 9aa0d225-1893-4fc0-8d5d-40cf890c4067 (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9e64b929-13c2-4e94-ba4f-96f15d4ba89a (dataset): DatasetConfig { id: 9e64b929-13c2-4e94-ba4f-96f15d4ba89a (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a0d4c349-9c98-4a73-a925-7147d15826af (dataset): DatasetConfig { id: a0d4c349-9c98-4a73-a925-7147d15826af (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a6d1c34e-ab1b-4003-ad9c-83c0ad244a7d (dataset): DatasetConfig { id: a6d1c34e-ab1b-4003-ad9c-83c0ad244a7d (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a8df2468-e878-45e6-832a-fddd2225182c (dataset): DatasetConfig { id: a8df2468-e878-45e6-832a-fddd2225182c (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_72e950cb-aef5-40dd-aa92-293783dc94a9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a984653e-3079-4570-a013-ae3c15e5ce07 (dataset): DatasetConfig { id: a984653e-3079-4570-a013-ae3c15e5ce07 (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_2306f05d-d638-4515-af92-dc4836424084\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, aca6795e-4c1e-43ed-913e-0306275b8e4b (dataset): DatasetConfig { id: aca6795e-4c1e-43ed-913e-0306275b8e4b (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_1dfeb85c-baf9-49e2-9020-eaca2cf6ce8c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b2a6fb9d-7c72-4a2a-8ea1-d8423ce57969 (dataset): DatasetConfig { id: b2a6fb9d-7c72-4a2a-8ea1-d8423ce57969 (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_9785445e-9baa-4024-8a89-6b305fb3e36a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b4c03dc8-848c-42c0-852c-45d1e019fdfe (dataset): DatasetConfig { id: b4c03dc8-848c-42c0-852c-45d1e019fdfe (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b667d6bf-cd95-46fe-a358-22481d6d053f (dataset): DatasetConfig { id: b667d6bf-cd95-46fe-a358-22481d6d053f (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cc2afb42-705b-4632-bd97-aa8aaec066a4 (dataset): DatasetConfig { id: cc2afb42-705b-4632-bd97-aa8aaec066a4 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cf2cf5d1-07e2-42cf-8928-1b07f767f047 (dataset): DatasetConfig { id: cf2cf5d1-07e2-42cf-8928-1b07f767f047 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, d2b8c34d-dc1a-46e0-96ec-d6982badb0e6 (dataset): DatasetConfig { id: d2b8c34d-dc1a-46e0-96ec-d6982badb0e6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_7b1c75dd-240b-4017-a160-edd46f1b3d00\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d73cd6fb-47af-40a8-a0fa-94b371a46ad3 (dataset): DatasetConfig { id: d73cd6fb-47af-40a8-a0fa-94b371a46ad3 (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d9e5b916-9c1b-4b59-b319-26e322fbdb61 (dataset): DatasetConfig { id: d9e5b916-9c1b-4b59-b319-26e322fbdb61 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dbda3008-9dab-4080-ad6f-45c5fb06ba02 (dataset): DatasetConfig { id: dbda3008-9dab-4080-ad6f-45c5fb06ba02 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, dc0beae4-4e1b-420d-8bde-db32c20e0766 (dataset): DatasetConfig { id: dc0beae4-4e1b-420d-8bde-db32c20e0766 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_445c7eec-d66d-460b-9a24-27198fcf6179\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e0ba913d-d324-479a-9c68-11b2e74233ff (dataset): DatasetConfig { id: e0ba913d-d324-479a-9c68-11b2e74233ff (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e3405a87-6129-4340-b145-44b361ae589f (dataset): DatasetConfig { id: e3405a87-6129-4340-b145-44b361ae589f (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e76a48c5-4ab0-45e0-8a91-7b4c18c5c3b0 (dataset): DatasetConfig { id: e76a48c5-4ab0-45e0-8a91-7b4c18c5c3b0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f18c0c55-424d-4a31-a713-40eb48be250e (dataset): DatasetConfig { id: f18c0c55-424d-4a31-a713-40eb48be250e (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f42f9875-f490-4a91-ac26-ac14081052dd (dataset): DatasetConfig { id: f42f9875-f490-4a91-ac26-ac14081052dd (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f4a0219f-54a0-4ca0-b405-241fecf8f740 (dataset): DatasetConfig { id: f4a0219f-54a0-4ca0-b405-241fecf8f740 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_20d7411f-1498-441e-89c3-f4071a3d9fb5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f988315b-a9ab-4cd6-8440-4fb757c7615b (dataset): DatasetConfig { id: f988315b-a9ab-4cd6-8440-4fb757c7615b (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_1cc9eeb4-0d42-4938-8c8b-a21df5e97007\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fc2fb1b2-6d4b-40c4-ab61-aaa08209b143 (dataset): DatasetConfig { id: fc2fb1b2-6d4b-40c4-ab61-aaa08209b143 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_77622b51-9ad8-477c-8380-8cae282506aa\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.296087147Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:460"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.296113658Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:46149"} {"msg":"datasets ensure","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.296630138Z","hostname":"sock","pid":1566,"sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:862"} {"msg":"Received DatasetsEnsure { config: DatasetsConfig { generation: Generation(1), datasets: {02bdb46f-1f7a-4495-8e8a-a1c8f0a03bc0 (dataset): DatasetConfig { id: 02bdb46f-1f7a-4495-8e8a-a1c8f0a03bc0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 08299bfb-0800-4ca8-9b8d-9b576960e166 (dataset): DatasetConfig { id: 08299bfb-0800-4ca8-9b8d-9b576960e166 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 09cc2927-2cf5-4413-8be0-f1ddbc79984d (dataset): DatasetConfig { id: 09cc2927-2cf5-4413-8be0-f1ddbc79984d (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 10dda82e-9265-49e8-b8f3-2a4a9c4c164b (dataset): DatasetConfig { id: 10dda82e-9265-49e8-b8f3-2a4a9c4c164b (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_7acb7f99-08f6-422b-b4ef-5456a5a44c23\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 125f36d7-301e-406a-bc11-db67930c216a (dataset): DatasetConfig { id: 125f36d7-301e-406a-bc11-db67930c216a (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 1356fa90-9862-4602-8bec-987f6eefba47 (dataset): DatasetConfig { id: 1356fa90-9862-4602-8bec-987f6eefba47 (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1542d45f-d5fe-49f7-aac2-c5dc5b736508 (dataset): DatasetConfig { id: 1542d45f-d5fe-49f7-aac2-c5dc5b736508 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1ba3c1dd-2484-4777-8624-adf9adabad51 (dataset): DatasetConfig { id: 1ba3c1dd-2484-4777-8624-adf9adabad51 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_70eb21d3-150c-4d21-bcc3-bbf3bd06c293\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1bd3a984-3b13-48a9-9fd9-5a85e418f46b (dataset): DatasetConfig { id: 1bd3a984-3b13-48a9-9fd9-5a85e418f46b (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2385dbc1-3195-45e0-8662-43c4a515014c (dataset): DatasetConfig { id: 2385dbc1-3195-45e0-8662-43c4a515014c (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 25705b62-3c7a-45bd-9cba-99bd5767da59 (dataset): DatasetConfig { id: 25705b62-3c7a-45bd-9cba-99bd5767da59 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_d5087ea8-b098-443c-b9ed-af8abfbf27e8\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 26790d95-ebdd-44fd-86e5-9dd4ba3e4f21 (dataset): DatasetConfig { id: 26790d95-ebdd-44fd-86e5-9dd4ba3e4f21 (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 26e0b6b0-ce70-43ca-980b-0cf12a292bcf (dataset): DatasetConfig { id: 26e0b6b0-ce70-43ca-980b-0cf12a292bcf (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 2d1e47ac-ebac-444a-b49e-9d3573199828 (dataset): DatasetConfig { id: 2d1e47ac-ebac-444a-b49e-9d3573199828 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 31e811be-26c1-4ef9-b32e-05600feef342 (dataset): DatasetConfig { id: 31e811be-26c1-4ef9-b32e-05600feef342 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_33c09be7-40a9-46e5-829a-6af4b245eb8d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 338b98ec-77e1-415d-8738-e1eea139be02 (dataset): DatasetConfig { id: 338b98ec-77e1-415d-8738-e1eea139be02 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_6bb5054e-2e33-47c5-8591-838209d609a0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3890ce90-eb11-4f7f-b73f-41bc8c3549cd (dataset): DatasetConfig { id: 3890ce90-eb11-4f7f-b73f-41bc8c3549cd (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 3a270f45-f343-4a4c-a795-e1d1d4026117 (dataset): DatasetConfig { id: 3a270f45-f343-4a4c-a795-e1d1d4026117 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a605806-3dd8-48f6-a271-4921a40cab01 (dataset): DatasetConfig { id: 3a605806-3dd8-48f6-a271-4921a40cab01 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4801adcb-51da-4c43-a6b8-74c9ab7d469e (dataset): DatasetConfig { id: 4801adcb-51da-4c43-a6b8-74c9ab7d469e (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 53e73ca6-22a3-4760-8d8c-13ba65fd0138 (dataset): DatasetConfig { id: 53e73ca6-22a3-4760-8d8c-13ba65fd0138 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 57924c57-7a89-42cf-b574-6af7898fbf20 (dataset): DatasetConfig { id: 57924c57-7a89-42cf-b574-6af7898fbf20 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 57d1f209-4bbf-4b28-8fc6-71af4aa89766 (dataset): DatasetConfig { id: 57d1f209-4bbf-4b28-8fc6-71af4aa89766 (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 57d5be5f-b1fb-493e-abd9-49787f94f8bd (dataset): DatasetConfig { id: 57d5be5f-b1fb-493e-abd9-49787f94f8bd (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_6a23c2a5-006e-4c29-a54c-390deecea882\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 58434d8e-917a-40ea-a69c-83af306968fe (dataset): DatasetConfig { id: 58434d8e-917a-40ea-a69c-83af306968fe (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_60daa419-d06f-44da-8437-507d9480bdc5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5b27c472-485e-4e9d-9aba-7cd17c6bd10d (dataset): DatasetConfig { id: 5b27c472-485e-4e9d-9aba-7cd17c6bd10d (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5e7c3ef9-e5f3-4f5b-afd5-97a14d72a56c (dataset): DatasetConfig { id: 5e7c3ef9-e5f3-4f5b-afd5-97a14d72a56c (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 626f8ac1-7e2f-4555-a521-96f6d9d1c49f (dataset): DatasetConfig { id: 626f8ac1-7e2f-4555-a521-96f6d9d1c49f (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 628dbe7b-bcea-4ffc-9f4e-c56ad9549e5e (dataset): DatasetConfig { id: 628dbe7b-bcea-4ffc-9f4e-c56ad9549e5e (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 63082ad4-1646-44cb-a619-f99202ef3e1d (dataset): DatasetConfig { id: 63082ad4-1646-44cb-a619-f99202ef3e1d (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_2e431855-2d60-4eca-bbca-519c0a7d8124\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6cf98f37-21a1-4ff0-b9a6-3d0efc3ee8be (dataset): DatasetConfig { id: 6cf98f37-21a1-4ff0-b9a6-3d0efc3ee8be (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_1bfa1410-b334-4a2b-ac4e-77fa9c04ad35\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6ee13928-1186-4e68-ae12-f5f0880b9bda (dataset): DatasetConfig { id: 6ee13928-1186-4e68-ae12-f5f0880b9bda (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 749db2ac-f07c-459b-8dfe-2e14e25a342c (dataset): DatasetConfig { id: 749db2ac-f07c-459b-8dfe-2e14e25a342c (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 758f88cf-7734-4193-a836-12f01f4bbff8 (dataset): DatasetConfig { id: 758f88cf-7734-4193-a836-12f01f4bbff8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_51618710-2d9e-4911-bfc3-577586e96ca3\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 787d82fb-5c78-42bb-8d95-7bd9d94d3c37 (dataset): DatasetConfig { id: 787d82fb-5c78-42bb-8d95-7bd9d94d3c37 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7fd0b8db-fc34-4652-b220-93eaa7beee17 (dataset): DatasetConfig { id: 7fd0b8db-fc34-4652-b220-93eaa7beee17 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8948f738-5705-4437-8259-8bb3c045db11 (dataset): DatasetConfig { id: 8948f738-5705-4437-8259-8bb3c045db11 (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8e32d5e5-5fbe-40f4-87a8-39a7939aa467 (dataset): DatasetConfig { id: 8e32d5e5-5fbe-40f4-87a8-39a7939aa467 (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9237f071-5643-491d-9e34-1f622d5fe205 (dataset): DatasetConfig { id: 9237f071-5643-491d-9e34-1f622d5fe205 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 93bcddc6-63cc-4c1b-a60f-47b0fb416c23 (dataset): DatasetConfig { id: 93bcddc6-63cc-4c1b-a60f-47b0fb416c23 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 99c77c44-c34f-4aa4-9f16-3e4edb45d1ec (dataset): DatasetConfig { id: 99c77c44-c34f-4aa4-9f16-3e4edb45d1ec (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9aa0d225-1893-4fc0-8d5d-40cf890c4067 (dataset): DatasetConfig { id: 9aa0d225-1893-4fc0-8d5d-40cf890c4067 (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9e64b929-13c2-4e94-ba4f-96f15d4ba89a (dataset): DatasetConfig { id: 9e64b929-13c2-4e94-ba4f-96f15d4ba89a (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a0d4c349-9c98-4a73-a925-7147d15826af (dataset): DatasetConfig { id: a0d4c349-9c98-4a73-a925-7147d15826af (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a6d1c34e-ab1b-4003-ad9c-83c0ad244a7d (dataset): DatasetConfig { id: a6d1c34e-ab1b-4003-ad9c-83c0ad244a7d (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a8df2468-e878-45e6-832a-fddd2225182c (dataset): DatasetConfig { id: a8df2468-e878-45e6-832a-fddd2225182c (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_72e950cb-aef5-40dd-aa92-293783dc94a9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a984653e-3079-4570-a013-ae3c15e5ce07 (dataset): DatasetConfig { id: a984653e-3079-4570-a013-ae3c15e5ce07 (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_2306f05d-d638-4515-af92-dc4836424084\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, aca6795e-4c1e-43ed-913e-0306275b8e4b (dataset): DatasetConfig { id: aca6795e-4c1e-43ed-913e-0306275b8e4b (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_1dfeb85c-baf9-49e2-9020-eaca2cf6ce8c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b2a6fb9d-7c72-4a2a-8ea1-d8423ce57969 (dataset): DatasetConfig { id: b2a6fb9d-7c72-4a2a-8ea1-d8423ce57969 (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_9785445e-9baa-4024-8a89-6b305fb3e36a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b4c03dc8-848c-42c0-852c-45d1e019fdfe (dataset): DatasetConfig { id: b4c03dc8-848c-42c0-852c-45d1e019fdfe (dataset), name: DatasetName { pool_name: ZpoolName { id: b0ef393a-1bdf-419f-872d-787b91138b01 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b667d6bf-cd95-46fe-a358-22481d6d053f (dataset): DatasetConfig { id: b667d6bf-cd95-46fe-a358-22481d6d053f (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cc2afb42-705b-4632-bd97-aa8aaec066a4 (dataset): DatasetConfig { id: cc2afb42-705b-4632-bd97-aa8aaec066a4 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cf2cf5d1-07e2-42cf-8928-1b07f767f047 (dataset): DatasetConfig { id: cf2cf5d1-07e2-42cf-8928-1b07f767f047 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, d2b8c34d-dc1a-46e0-96ec-d6982badb0e6 (dataset): DatasetConfig { id: d2b8c34d-dc1a-46e0-96ec-d6982badb0e6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_7b1c75dd-240b-4017-a160-edd46f1b3d00\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d73cd6fb-47af-40a8-a0fa-94b371a46ad3 (dataset): DatasetConfig { id: d73cd6fb-47af-40a8-a0fa-94b371a46ad3 (dataset), name: DatasetName { pool_name: ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d9e5b916-9c1b-4b59-b319-26e322fbdb61 (dataset): DatasetConfig { id: d9e5b916-9c1b-4b59-b319-26e322fbdb61 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dbda3008-9dab-4080-ad6f-45c5fb06ba02 (dataset): DatasetConfig { id: dbda3008-9dab-4080-ad6f-45c5fb06ba02 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, dc0beae4-4e1b-420d-8bde-db32c20e0766 (dataset): DatasetConfig { id: dc0beae4-4e1b-420d-8bde-db32c20e0766 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_445c7eec-d66d-460b-9a24-27198fcf6179\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e0ba913d-d324-479a-9c68-11b2e74233ff (dataset): DatasetConfig { id: e0ba913d-d324-479a-9c68-11b2e74233ff (dataset), name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e3405a87-6129-4340-b145-44b361ae589f (dataset): DatasetConfig { id: e3405a87-6129-4340-b145-44b361ae589f (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e76a48c5-4ab0-45e0-8a91-7b4c18c5c3b0 (dataset): DatasetConfig { id: e76a48c5-4ab0-45e0-8a91-7b4c18c5c3b0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0ed2433e-119b-40a8-84d1-13f35981a5d0 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f18c0c55-424d-4a31-a713-40eb48be250e (dataset): DatasetConfig { id: f18c0c55-424d-4a31-a713-40eb48be250e (dataset), name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f42f9875-f490-4a91-ac26-ac14081052dd (dataset): DatasetConfig { id: f42f9875-f490-4a91-ac26-ac14081052dd (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f4a0219f-54a0-4ca0-b405-241fecf8f740 (dataset): DatasetConfig { id: f4a0219f-54a0-4ca0-b405-241fecf8f740 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_20d7411f-1498-441e-89c3-f4071a3d9fb5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f988315b-a9ab-4cd6-8440-4fb757c7615b (dataset): DatasetConfig { id: f988315b-a9ab-4cd6-8440-4fb757c7615b (dataset), name: DatasetName { pool_name: ZpoolName { id: 94913555-b694-4ecc-9c16-7af8c3b569ae (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_1cc9eeb4-0d42-4938-8c8b-a21df5e97007\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fc2fb1b2-6d4b-40c4-ab61-aaa08209b143 (dataset): DatasetConfig { id: fc2fb1b2-6d4b-40c4-ab61-aaa08209b143 (dataset), name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_77622b51-9ad8-477c-8380-8cae282506aa\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.296874373Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.299403166Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.299516618Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested datasets', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.299522308Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:905"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.299534598Z","hostname":"sock","pid":1566,"name":"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.299556339Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/clickhouse\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.385393189Z","hostname":"sock","pid":1566,"name":"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.38543296Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.472721051Z","hostname":"sock","pid":1566,"name":"oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.472756551Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.558904398Z","hostname":"sock","pid":1566,"name":"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_crucible_7acb7f99-08f6-422b-b4ef-5456a5a44c23","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.558941999Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_crucible_7acb7f99-08f6-422b-b4ef-5456a5a44c23\"), full_name: \"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_crucible_7acb7f99-08f6-422b-b4ef-5456a5a44c23\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.654436809Z","hostname":"sock","pid":1566,"name":"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.65447349Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/debug\"), full_name: \"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.719760306Z","hostname":"sock","pid":1566,"name":"oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.719798977Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone\"), full_name: \"oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.787098015Z","hostname":"sock","pid":1566,"name":"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.787134836Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\"), full_name: \"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.887875854Z","hostname":"sock","pid":1566,"name":"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_crucible_pantry_70eb21d3-150c-4d21-bcc3-bbf3bd06c293","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.887914905Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_crucible_pantry_70eb21d3-150c-4d21-bcc3-bbf3bd06c293\"), full_name: \"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_crucible_pantry_70eb21d3-150c-4d21-bcc3-bbf3bd06c293\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.987011869Z","hostname":"sock","pid":1566,"name":"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:04.98704947Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone\"), full_name: \"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.054019791Z","hostname":"sock","pid":1566,"name":"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.054055092Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:05.064390865Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.143080658Z","hostname":"sock","pid":1566,"name":"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_crucible_d5087ea8-b098-443c-b9ed-af8abfbf27e8","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.143118099Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_crucible_d5087ea8-b098-443c-b9ed-af8abfbf27e8\"), full_name: \"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_crucible_d5087ea8-b098-443c-b9ed-af8abfbf27e8\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.240392085Z","hostname":"sock","pid":1566,"name":"oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.240438316Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/debug\"), full_name: \"oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.307296515Z","hostname":"sock","pid":1566,"name":"oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.307334526Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/debug\"), full_name: \"oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.374298987Z","hostname":"sock","pid":1566,"name":"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.374339758Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/debug\"), full_name: \"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.441436782Z","hostname":"sock","pid":1566,"name":"oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/zone/oxz_crucible_33c09be7-40a9-46e5-829a-6af4b245eb8d","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.441478263Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/zone/oxz_crucible_33c09be7-40a9-46e5-829a-6af4b245eb8d\"), full_name: \"oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/zone/oxz_crucible_33c09be7-40a9-46e5-829a-6af4b245eb8d\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.539164568Z","hostname":"sock","pid":1566,"name":"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_crucible_6bb5054e-2e33-47c5-8591-838209d609a0","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.539206419Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_crucible_6bb5054e-2e33-47c5-8591-838209d609a0\"), full_name: \"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_crucible_6bb5054e-2e33-47c5-8591-838209d609a0\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.639393886Z","hostname":"sock","pid":1566,"name":"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.639438476Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/debug\"), full_name: \"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.708318437Z","hostname":"sock","pid":1566,"name":"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.708359158Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone\"), full_name: \"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.776829541Z","hostname":"sock","pid":1566,"name":"oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.776869931Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/zone\"), full_name: \"oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.843809342Z","hostname":"sock","pid":1566,"name":"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.843859373Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5\"), full_name: \"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.942966177Z","hostname":"sock","pid":1566,"name":"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:05.943010568Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone\"), full_name: \"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.011856638Z","hostname":"sock","pid":1566,"name":"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.011896779Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:06.067929485Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.100241801Z","hostname":"sock","pid":1566,"name":"oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.100281002Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/debug\"), full_name: \"oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.169611332Z","hostname":"sock","pid":1566,"name":"oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone/oxz_nexus_6a23c2a5-006e-4c29-a54c-390deecea882","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.169648723Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone/oxz_nexus_6a23c2a5-006e-4c29-a54c-390deecea882\"), full_name: \"oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone/oxz_nexus_6a23c2a5-006e-4c29-a54c-390deecea882\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.269698527Z","hostname":"sock","pid":1566,"name":"oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone/oxz_crucible_pantry_60daa419-d06f-44da-8437-507d9480bdc5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.269739897Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone/oxz_crucible_pantry_60daa419-d06f-44da-8437-507d9480bdc5\"), full_name: \"oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone/oxz_crucible_pantry_60daa419-d06f-44da-8437-507d9480bdc5\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.369286181Z","hostname":"sock","pid":1566,"name":"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.369329262Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\"), full_name: \"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.468104069Z","hostname":"sock","pid":1566,"name":"oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.46814651Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone\"), full_name: \"oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.536346807Z","hostname":"sock","pid":1566,"name":"oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.536385868Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.625682439Z","hostname":"sock","pid":1566,"name":"oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.62572682Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b0ef393a-1bdf-419f-872d-787b91138b01/crypt/debug\"), full_name: \"oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.692940457Z","hostname":"sock","pid":1566,"name":"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_oximeter_2e431855-2d60-4eca-bbca-519c0a7d8124","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.692987148Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_oximeter_2e431855-2d60-4eca-bbca-519c0a7d8124\"), full_name: \"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_oximeter_2e431855-2d60-4eca-bbca-519c0a7d8124\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.794079923Z","hostname":"sock","pid":1566,"name":"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_crucible_1bfa1410-b334-4a2b-ac4e-77fa9c04ad35","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.794119544Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_crucible_1bfa1410-b334-4a2b-ac4e-77fa9c04ad35\"), full_name: \"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_crucible_1bfa1410-b334-4a2b-ac4e-77fa9c04ad35\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.893869831Z","hostname":"sock","pid":1566,"name":"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.893907132Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.984375838Z","hostname":"sock","pid":1566,"name":"oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:06.984415839Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:07.070219629Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.073060288Z","hostname":"sock","pid":1566,"name":"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_crucible_pantry_51618710-2d9e-4911-bfc3-577586e96ca3","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.073071808Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_crucible_pantry_51618710-2d9e-4911-bfc3-577586e96ca3\"), full_name: \"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_crucible_pantry_51618710-2d9e-4911-bfc3-577586e96ca3\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.172159362Z","hostname":"sock","pid":1566,"name":"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.172202733Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29\"), full_name: \"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.271119283Z","hostname":"sock","pid":1566,"name":"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.271165964Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.362832205Z","hostname":"sock","pid":1566,"name":"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.362882616Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.448925241Z","hostname":"sock","pid":1566,"name":"oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.448964212Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone\"), full_name: \"oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.515990544Z","hostname":"sock","pid":1566,"name":"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.516029135Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.605097212Z","hostname":"sock","pid":1566,"name":"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.605139823Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.69858029Z","hostname":"sock","pid":1566,"name":"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.698628191Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.782616864Z","hostname":"sock","pid":1566,"name":"oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.782659925Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.866153527Z","hostname":"sock","pid":1566,"name":"oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.866191718Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.950556948Z","hostname":"sock","pid":1566,"name":"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:07.950596989Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.040456702Z","hostname":"sock","pid":1566,"name":"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.040500353Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:08.072103885Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.12618183Z","hostname":"sock","pid":1566,"name":"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_nexus_72e950cb-aef5-40dd-aa92-293783dc94a9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.126223061Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_nexus_72e950cb-aef5-40dd-aa92-293783dc94a9\"), full_name: \"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_nexus_72e950cb-aef5-40dd-aa92-293783dc94a9\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.222495327Z","hostname":"sock","pid":1566,"name":"oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone/oxz_crucible_2306f05d-d638-4515-af92-dc4836424084","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.222539768Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone/oxz_crucible_2306f05d-d638-4515-af92-dc4836424084\"), full_name: \"oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone/oxz_crucible_2306f05d-d638-4515-af92-dc4836424084\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.32297957Z","hostname":"sock","pid":1566,"name":"oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone/oxz_crucible_1dfeb85c-baf9-49e2-9020-eaca2cf6ce8c","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.323021981Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone/oxz_crucible_1dfeb85c-baf9-49e2-9020-eaca2cf6ce8c\"), full_name: \"oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone/oxz_crucible_1dfeb85c-baf9-49e2-9020-eaca2cf6ce8c\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.421792048Z","hostname":"sock","pid":1566,"name":"oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone/oxz_crucible_9785445e-9baa-4024-8a89-6b305fb3e36a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.421833549Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone/oxz_crucible_9785445e-9baa-4024-8a89-6b305fb3e36a\"), full_name: \"oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone/oxz_crucible_9785445e-9baa-4024-8a89-6b305fb3e36a\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.518746178Z","hostname":"sock","pid":1566,"name":"oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.518791149Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone\"), full_name: \"oxp_b0ef393a-1bdf-419f-872d-787b91138b01/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.585008445Z","hostname":"sock","pid":1566,"name":"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.585051835Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.675281427Z","hostname":"sock","pid":1566,"name":"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.675336998Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone\"), full_name: \"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.744149447Z","hostname":"sock","pid":1566,"name":"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.744189168Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/debug\"), full_name: \"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.811104838Z","hostname":"sock","pid":1566,"name":"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_external_dns_7b1c75dd-240b-4017-a160-edd46f1b3d00","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.811147999Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_external_dns_7b1c75dd-240b-4017-a160-edd46f1b3d00\"), full_name: \"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_external_dns_7b1c75dd-240b-4017-a160-edd46f1b3d00\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.908510018Z","hostname":"sock","pid":1566,"name":"oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone/oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:08.908557779Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone/oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891\"), full_name: \"oxp_d634c5ac-cef9-463f-93e8-f083982a2a97/crypt/zone/oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.031538865Z","hostname":"sock","pid":1566,"name":"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.031584686Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\"), full_name: \"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:09.073367748Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.134300545Z","hostname":"sock","pid":1566,"name":"oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.134335145Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/debug\"), full_name: \"oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.204453442Z","hostname":"sock","pid":1566,"name":"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_nexus_445c7eec-d66d-460b-9a24-27198fcf6179","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.204489563Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_nexus_445c7eec-d66d-460b-9a24-27198fcf6179\"), full_name: \"oxp_5b47f8d5-0da4-4301-b2f1-2346f32d252b/crypt/zone/oxz_nexus_445c7eec-d66d-460b-9a24-27198fcf6179\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.306138909Z","hostname":"sock","pid":1566,"name":"oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone/oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.3061848Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone/oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\"), full_name: \"oxp_340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f/crypt/zone/oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.407541911Z","hostname":"sock","pid":1566,"name":"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.407579682Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f\"), full_name: \"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.50641302Z","hostname":"sock","pid":1566,"name":"oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.506451791Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_0ed2433e-119b-40a8-84d1-13f35981a5d0/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.595654621Z","hostname":"sock","pid":1566,"name":"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.595696152Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\"), full_name: \"oxp_bf132071-bc64-4b2d-9aee-c10dc030c6b7/crypt/zone/oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.701675168Z","hostname":"sock","pid":1566,"name":"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.701716369Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.794325479Z","hostname":"sock","pid":1566,"name":"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_clickhouse_20d7411f-1498-441e-89c3-f4071a3d9fb5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.79437173Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_clickhouse_20d7411f-1498-441e-89c3-f4071a3d9fb5\"), full_name: \"oxp_1a77147c-c146-4e56-8066-57b7b2b29250/crypt/zone/oxz_clickhouse_20d7411f-1498-441e-89c3-f4071a3d9fb5\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.891501254Z","hostname":"sock","pid":1566,"name":"oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone/oxz_crucible_1cc9eeb4-0d42-4938-8c8b-a21df5e97007","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.891541335Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone/oxz_crucible_1cc9eeb4-0d42-4938-8c8b-a21df5e97007\"), full_name: \"oxp_94913555-b694-4ecc-9c16-7af8c3b569ae/crypt/zone/oxz_crucible_1cc9eeb4-0d42-4938-8c8b-a21df5e97007\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:09.903585813Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12768160433890941050"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:09.903707275Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14686322292127562160"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.989119357Z","hostname":"sock","pid":1566,"name":"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_external_dns_77622b51-9ad8-477c-8380-8cae282506aa","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:09.989157718Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_external_dns_77622b51-9ad8-477c-8380-8cae282506aa\"), full_name: \"oxp_c86b98b9-5437-4bb7-b77f-625c07ff921c/crypt/zone/oxz_external_dns_77622b51-9ad8-477c-8380-8cae282506aa\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:10.075765624Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.089924827Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.093550671Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"datasets ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.093853068Z","hostname":"sock","pid":1566,"sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:864"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.0939931Z","hostname":"sock","pid":1566,"uri":"/datasets","method":"PUT","req_id":"e2721a10-c03a-4533-8649-4e91ea512385","remote_addr":"[fd00:1122:3344:101::1]:46149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5797743,"response_code":"200"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.116232529Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4292"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.11624625Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4302","gateway":"fd00:1122:3344:101::1"} {"msg":"Default route already exists","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.123148742Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4316","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.123184013Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4381"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29 (service), filesystem_pool: Some(ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 8cda43db-36a7-4db9-aa46-224c2e7183f5 (service), filesystem_pool: Some(ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f (service), filesystem_pool: Some(ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.133522346Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.13370101Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:34250"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.133959365Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.133965615Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cda43db-36a7-4db9-aa46-224c2e7183f5, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.170333465Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.170367086Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 8cda43db-36a7-4db9-aa46-224c2e7183f5, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.2035566Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.203593541Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.239240167Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.239272187Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.239283747Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.239338109Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.279585889Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4447"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.27966179Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4474"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.290274119Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.2903006Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.29030469Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.325073617Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4487"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.325098298Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4333"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.351508032Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.3557419Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.362654232Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.373280861Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.378422527Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.381334098Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4375"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.381394549Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4454","rack_subnet":"fd00:1122:3344::"} {"msg":"Installing Omicron zone: oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.384712217Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.404171618Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4465"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.404205639Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4507"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.491416288Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4539"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.491458249Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4545"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.604675854Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4586"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.604750336Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.642088296Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:420"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e358e662-7828-4f7f-85a4-21e089483ee6)","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:10.642526845Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.683851248Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.68396294Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:762"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.684584723Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:765","uptime":"1"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.684611743Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:444","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: Some(\n None,\n ),\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n tx_eq: None,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.731287156Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.788962206Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789025657Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789038097Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789045097Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789054328Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789061748Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789068358Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789078088Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789084758Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789101859Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789108809Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789115329Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789122019Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789128569Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789134799Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.78913961Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.789144519Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.78914923Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.78915384Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.78915835Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.78916303Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.78917919Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:10.78918413Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e358e662-7828-4f7f-85a4-21e089483ee6)","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:11.077994227Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:11.268472106Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14076285699142498485"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:11.268591799Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15791617063128661124"} {"msg":"Profile for oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:12.01522754Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:12.015277441Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:12.494249161Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:12.971201519Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:12.97127813Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:12.971390662Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:12.975394465Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Booting oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:12.975451616Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:12.979884758Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29\") failed: Property not found. retry in 34.28747ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:13.106468829Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f\") failed: Property not found. retry in 43.736626ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:13.112712517Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5\") failed: Property not found. retry in 60.921211ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:13.11814537Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29\") failed: Property not found. retry in 126.282121ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:13.148349273Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f\") failed: Property not found. retry in 97.030703ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:13.16521741Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5\") failed: Property not found. retry in 109.643284ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:13.331100952Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f\") failed: Property not found. retry in 177.020697ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:13.503490998Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5\") failed: Property not found. retry in 112.686102ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:13.527021133Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29\") failed: Property not found. retry in 261.70165ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:13.604608304Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5\") failed: Property not found. retry in 418.659812ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:13.714766326Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f\") failed: Property not found. retry in 584.008776ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:13.737621337Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29\") failed: Property not found. retry in 548.140796ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:13.893411681Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5\") failed: Property not found. retry in 993.42979ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:14.143972809Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f\") failed: Property not found. retry in 756.716093ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:14.331995928Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29\") failed: Property not found. retry in 511.748683ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:14.452382891Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29\") failed: Property not found. retry in 1.421549307s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:14.973404338Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f\") failed: Property not found. retry in 1.418229717s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:15.098974208Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5\") failed: Property not found. retry in 977.984982ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:15.146425757Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5\") failed: Property not found. retry in 691.403821ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:16.133929446Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29\") failed: Property not found. retry in 633.582444ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:16.406118071Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f\") failed: Property not found. retry in 1.353355823s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:16.529260611Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5\") failed: Property not found. retry in 1.335604394s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:16.835043938Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29\") failed: Property not found. retry in 758.311467ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:17.048644384Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29\") failed: Property not found. retry in 1.094720197s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:17.816285288Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f\") failed: Property not found. retry in 716.258683ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:17.890809896Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5\") failed: Property not found. retry in 728.140452ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:18.18048236Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f\") failed: Property not found. retry in 584.775374ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:18.61578656Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5\") failed: Property not found. retry in 915.913916ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:18.918454933Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29\") failed: Property not found. retry in 897.230337ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:18.929405809Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f\") failed: Property not found. retry in 1.072919887s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:19.210309433Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:19.905179566Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12768160433890941050"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:19.905305469Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14686322292127562160"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:20.303075453Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_internal_dns_f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:20.303119704Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_internal_dns_8cda43db-36a7-4db9-aa46-224c2e7183f5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:20.303127154Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_internal_dns_0b3f25f2-8c47-4ce0-a3b3-ead306097f29"} {"msg":"Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:20.303132894Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:20.30341627Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:20.303695366Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"c988513f-a8ce-42a7-9424-6aebb2b3cd10","remote_addr":"[fd00:1122:3344:101::1]:34250","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":10169868,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:20.305583225Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:21.270906166Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14076285699142498485"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:21.271020849Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15791617063128661124"} {"msg":"failed to write DNS configuration (will retry in 70.332979ms)","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:21.392227599Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:678","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config)"} {"msg":"failed to write DNS configuration (will retry in 144.342656ms)","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:21.46355645Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:678","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.659202596Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.738870079Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.819337569Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.81935125Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.852189447Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.854631027Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.854645567Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29 (service), filesystem_pool: Some(ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 8cda43db-36a7-4db9-aa46-224c2e7183f5 (service), filesystem_pool: Some(ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f (service), filesystem_pool: Some(ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 931fa059-cc81-4b86-be35-dae0f4c5b891 (service), filesystem_pool: Some(ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a3b3f151-c7d2-4d8a-8722-84fd46ba2ae6, kind: Service { id: 931fa059-cc81-4b86-be35-dae0f4c5b891 }, name: Name(\"ntp-931fa059-cc81-4b86-be35-dae0f4c5b891\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 229])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.885258589Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.885390312Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:40568"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.885553965Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.885564215Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cda43db-36a7-4db9-aa46-224c2e7183f5, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.931920832Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.931959452Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 8cda43db-36a7-4db9-aa46-224c2e7183f5, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.975255725Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:21.975304476Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.019902556Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.019962477Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.0200794Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.020162841Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.031139558Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.063881273Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.064609008Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.064627438Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.094532585Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 219, 229]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.094548466Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1263","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.094974004Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.094978175Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.09523739Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.095630748Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.119167814Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.137070473Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.755482379Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:22.755573251Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891\") failed: Property not found. retry in 59.18818ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:22.892056186Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891\") failed: Property not found. retry in 128.323297ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:22.962581061Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891\") failed: Property not found. retry in 127.672678ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:23.10119423Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891\") failed: Property not found. retry in 436.663855ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:23.239648606Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891\") failed: Property not found. retry in 782.346022ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:23.686391401Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891\") failed: Property not found. retry in 1.206492554s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:24.478313536Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891\") failed: Property not found. retry in 704.855546ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:25.694851209Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891\") failed: Property not found. retry in 590.799432ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:26.409881248Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891\") failed: Property not found. retry in 1.006614826s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:27.009514867Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891\") failed: Property not found. retry in 890.919471ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:28.025473613Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:28.935912143Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891"} {"msg":"Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:28.935994745Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:28.936306641Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:28.936643768Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"5e22cc44-6d19-4e36-bdaa-05c64c48703d","remote_addr":"[fd00:1122:3344:101::1]:40568","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7051203,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:28.938919095Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:735"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:28.968517895Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:28.968669078Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:36488"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:28.978045912Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3900"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:28.978161304Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"556afa6e-37b0-40f5-97e8-15bf2eb5b903","remote_addr":"[fd00:1122:3344:101::1]:36488","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9330,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:28.979662285Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:29.14181837Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:29.141947933Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:57119"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:29.863666689Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13115064489608424063"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:29.866953127Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11430979086984911751"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:29.906259098Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12768160433890941050"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:29.906524873Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14686322292127562160"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:30.307053245Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17942048789459529530"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:31.272326656Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14076285699142498485"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:31.272452889Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15791617063128661124"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:33.15111567Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3900"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:33.151283843Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"b39957e3-c4d1-4487-b6d9-2a7ebd5c0461","remote_addr":"[fd00:1122:3344:101::1]:57119","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009265,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:33.153987849Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:33.357209971Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:33.357395694Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:49077"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:33.365201466Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3900"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:33.365315718Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"2662d1fa-bc8c-4b22-8b9b-e5bd0ce7c032","remote_addr":"[fd00:1122:3344:101::1]:49077","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":7834,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:33.368630906Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:33.511183667Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:33.51132107Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:63998"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:37.511273968Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"244.175567ms"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_931fa059-cc81-4b86-be35-dae0f4c5b891': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:37.522529666Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3900"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:37.52266811Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"25fae244-fe9a-48fa-a3d0-4b764c11320f","remote_addr":"[fd00:1122:3344:101::1]:63998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4007234,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:37.524846599Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:37.677929775Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:37.678083818Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:54361"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:37.687410662Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"462f45ae-fc51-4074-a6a9-fa5877f8e614","remote_addr":"[fd00:1122:3344:101::1]:54361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9248,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.085487872 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:37.689616819Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:37.689622735Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:37.948475654Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:37.948585251Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:58312"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:37.957547544Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"082c9671-cd28-48b7-a3e2-9a8f02db7361","remote_addr":"[fd00:1122:3344:101::1]:58312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8887,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.085169524 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:37.959342642Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:37.959348653Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:38.319937223Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:38.32009436Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:44376"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:38.328050557Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"ace71a56-55be-48e6-8e39-4e1bf9be8489","remote_addr":"[fd00:1122:3344:101::1]:44376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7820,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.084168293 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:38.329932979Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:38.329937582Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:38.629678786Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:38.629771375Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:64393"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:38.637933715Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"faffde44-d579-4e50-b982-73fff477dabd","remote_addr":"[fd00:1122:3344:101::1]:64393","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8052,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.083331518 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:38.639398599Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:38.639404074Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:38.949353554Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5828729199679799402"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:38.949471701Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9931914467758814075"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:39.056182631Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:39.056427553Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:40178"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:39.065172214Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"02dcb4d5-6576-4019-88a4-fc3068fdccd8","remote_addr":"[fd00:1122:3344:101::1]:40178","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8682,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568718.4096134, correction: 0.082337789 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:39.067223733Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:39.067229716Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:39.347421901Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:39.347546168Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:61026"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:39.355265545Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"06c1b513-2b08-47b1-b2d3-723a015b42ef","remote_addr":"[fd00:1122:3344:101::1]:61026","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7610,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568718.4096134, correction: 0.081554256 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:39.359335289Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:39.35934142Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:39.853491203Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:39.853601604Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:54133"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:39.861984487Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"ce5f2cdf-18ce-447c-8b00-60ce112387ea","remote_addr":"[fd00:1122:3344:101::1]:54133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8257,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568718.8964703, correction: 0.079777628 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:39.863425319Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:39.863430832Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:39.876886319Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13115064489608424063"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:39.879059616Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11430979086984911751"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:39.918476178Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12768160433890941050"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:39.918705064Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14686322292127562160"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:40.319723901Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17942048789459529530"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:40.347504223Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62257"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:40.347723641Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"f61ac20d-893c-4afa-950e-55d396f5acf3","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62257","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:40.728804841Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:40.728978583Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:60628"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:40.738753736Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"9a14dc66-79c9-46cc-b153-89d18f6518ef","remote_addr":"[fd00:1122:3344:101::1]:60628","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9660,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568718.8964703, correction: 0.077410877 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:40.740699758Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:40.740708514Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:41.290219652Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14076285699142498485"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:41.290341988Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15791617063128661124"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:41.576728584Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:41.576942979Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:61034"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:41.585984869Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"dc954721-f2bc-4b29-904d-b234b29f8324","remote_addr":"[fd00:1122:3344:101::1]:61034","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8944,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568720.7604134, correction: 0.075146243 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:41.589240578Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:41.589262591Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:42.499059058Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:42.499207871Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:62756"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:42.508061374Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"950cc70f-f9a5-4335-adec-5c8ee0030474","remote_addr":"[fd00:1122:3344:101::1]:62756","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8709,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568721.2434585, correction: 0.072281636 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:42.510146002Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:42.510157496Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:44.095376854Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:44.095514548Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:42808"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:44.104153016Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"82d44f9d-3dd4-490d-8a79-64f1eefb94c2","remote_addr":"[fd00:1122:3344:101::1]:42808","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8518,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568723.2554588, correction: 0.067650191 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:44.107313516Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:44.107320473Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:45.946705416Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:45.946902725Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:40085"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:45.955935871Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"f4409bdb-8ba0-4283-8a1e-a5dcea876c1f","remote_addr":"[fd00:1122:3344:101::1]:40085","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8941,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568725.2572372, correction: 0.062688097 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:45.957579406Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:45.957586719Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:47.058177921Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:47.05832558Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:38295"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:47.06724317Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"b482f227-909b-4d3c-8ccf-87ae0e89d471","remote_addr":"[fd00:1122:3344:101::1]:38295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8808,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568726.7274144, correction: 0.059760634 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:47.069558203Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:47.069564325Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:48.979618101Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5828729199679799402"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:48.979736163Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9931914467758814075"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:49.745221821Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:49.745383047Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:51907"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:49.754554159Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"7d7eb8f8-4990-49e4-a34e-349281f5582e","remote_addr":"[fd00:1122:3344:101::1]:51907","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9038,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568728.5917926, correction: 0.052509069 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:49.758305671Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:49.758316416Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:49.905888905Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13115064489608424063"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:49.908031858Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11430979086984911751"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:49.947481046Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12768160433890941050"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:49.947765745Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14686322292127562160"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:50.348494997Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17942048789459529530"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:51.320513193Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14076285699142498485"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-19T00:38:51.320626032Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15791617063128661124"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.448265314Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.44846805Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:62066"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.470200838Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"cf2a67ab-faeb-422c-ac85-dd4bb7d5e796","remote_addr":"[fd00:1122:3344:101::1]:62066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":21503,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1734568730.42558, correction: 0.045051727 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.474865988Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.474870656Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1392"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29 (service), filesystem_pool: Some(ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 8cda43db-36a7-4db9-aa46-224c2e7183f5 (service), filesystem_pool: Some(ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f (service), filesystem_pool: Some(ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 931fa059-cc81-4b86-be35-dae0f4c5b891 (service), filesystem_pool: Some(ZpoolName { id: d634c5ac-cef9-463f-93e8-f083982a2a97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a3b3f151-c7d2-4d8a-8722-84fd46ba2ae6, kind: Service { id: 931fa059-cc81-4b86-be35-dae0f4c5b891 }, name: Name(\"ntp-931fa059-cc81-4b86-be35-dae0f4c5b891\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 229])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 05bfdf27-226d-408e-8634-4cf7d9392bcf (service), filesystem_pool: Some(ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External } } } }, OmicronZoneConfig { id: a52f73c1-a1a5-4560-812f-18319626624c (service), filesystem_pool: Some(ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3cc04303-718d-447d-bc89-3a4154b619d7 (service), filesystem_pool: Some(ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 57ed5e38-59cc-4e8b-8338-2dceef12fef7 (service), filesystem_pool: Some(ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External } } } }, OmicronZoneConfig { id: 5c749b5a-3b6c-44a8-9cb5-f97707372cba (service), filesystem_pool: Some(ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.504757487Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.504874817Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:64045"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.505078239Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.505084493Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 0b3f25f2-8c47-4ce0-a3b3-ead306097f29, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cda43db-36a7-4db9-aa46-224c2e7183f5, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.543645026Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.543683689Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 8cda43db-36a7-4db9-aa46-224c2e7183f5, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.580657065Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.580767821Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f8d56782-d0db-41a9-97dd-7a3fa9f8ef1f, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05bfdf27-226d-408e-8634-4cf7d9392bcf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.61495829Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.61499341Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 05bfdf27-226d-408e-8634-4cf7d9392bcf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a77147c-c146-4e56-8066-57b7b2b29250 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a52f73c1-a1a5-4560-812f-18319626624c, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.653434083Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.653471959Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a52f73c1-a1a5-4560-812f-18319626624c, dataset_name: DatasetName { pool_name: ZpoolName { id: c86b98b9-5437-4bb7-b77f-625c07ff921c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3cc04303-718d-447d-bc89-3a4154b619d7, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.689353902Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.689395846Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 3cc04303-718d-447d-bc89-3a4154b619d7, dataset_name: DatasetName { pool_name: ZpoolName { id: bf132071-bc64-4b2d-9aee-c10dc030c6b7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57ed5e38-59cc-4e8b-8338-2dceef12fef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.726293944Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.726341551Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 57ed5e38-59cc-4e8b-8338-2dceef12fef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b47f8d5-0da4-4301-b2f1-2346f32d252b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c749b5a-3b6c-44a8-9cb5-f97707372cba, dataset_name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.76109364Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.76112979Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 5c749b5a-3b6c-44a8-9cb5-f97707372cba, dataset_name: DatasetName { pool_name: ZpoolName { id: 340fd2a3-1a04-45a1-bbdd-2d0c66b59e2f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.797062162Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.79715612Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.797490393Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.797613987Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:52.83497555Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"312.38578ms"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.878255354Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.878316302Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.878324908Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.878333477Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.878340181Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.967848496Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.972143043Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.976085268Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.980247571Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.984959253Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.989672982Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.99418198Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:52.998715629Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:53.002562481Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:53.00644968Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:55.791316378Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:55.791497135Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:55.83744559Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:55.837676374Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:55.895599239Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:55.895829407Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:55.914438057Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:55.914574681Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:55.9201041Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:38:55.920234837Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 56.339247ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:55.946171648Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 50.391227ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:55.995119487Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 68.782609ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.011706299Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 73.559923ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.061318863Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 50.685773ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.085587043Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 288.246784ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.233190583Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 159.618055ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.267341155Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 56.944482ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.282212287Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 62.227662ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.305993661Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 66.237825ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.336936522Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 212.397393ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.36268203Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 99.282639ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.389072688Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 82.787028ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.446259067Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 576.342207ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.582615662Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 492.094388ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.611745566Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 278.810744ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.638346458Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 295.053358ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.689978647Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 476.792795ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.821390203Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 373.941223ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:56.970263499Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 483.428865ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:57.058062439Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 942.633836ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:57.147655156Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 988.229996ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:57.199538755Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 976.056875ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:57.358553467Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 547.44845ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:57.417687033Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 1.109788841s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:57.628444478Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 1.272722635s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:57.997784727Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 688.182848ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:58.124162758Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 621.141615ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:58.223039392Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 596.635072ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:58.463898258Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 1.118083952s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:58.753329855Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 1.389174584s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:58.824621553Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 815.896591ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:58.855049234Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 1.341340994s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:59.073634819Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 1.290770487s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:59.285542802Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 1.411496344s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:59.684576105Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 951.78852ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:38:59.885040375Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 1.32034073s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:00.226275856Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 701.567968ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:00.428438408Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 1.281077291s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:00.589595934Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 523.23583ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:00.848352497Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 766.008185ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:01.108032244Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 987.838072ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:01.13971519Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 1.118174487s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:01.381710058Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 724.398001ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:01.558878495Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 639.230909ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:01.884179919Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 875.503604ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:01.891911766Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 675.809972ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:02.139952443Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 872.141614ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:02.29481713Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 562.821061ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:02.513672878Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 591.00212ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:02.534231809Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 815.058507ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:02.778611928Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 614.263022ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:02.825243667Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 657.578926ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:03.086771136Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 832.009002ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:03.13609328Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 809.203805ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:03.177945605Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 1.150389777s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:03.450782188Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 986.288269ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:03.60507307Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 703.520231ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:03.75515915Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 682.89599ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:03.977858771Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 945.60907ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:03.99724732Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 1.317047983s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:04.469116576Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 688.912846ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:04.602638288Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 1.113345565s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:04.611571592Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 854.597119ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:04.671384004Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 780.956806ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:04.953551041Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 1.297095538s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:05.30299393Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 1.258817908s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:05.538136505Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 1.34151972s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:05.73680654Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 665.470468ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:05.745146916Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 981.782405ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:05.798565937Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 1.282156565s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:06.421396141Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 972.986663ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:06.611837718Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 1.224176343s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:06.791395513Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 1.491980415s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:06.807786988Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 1.473816005s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:07.090608465Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 1.042014751s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:07.596678323Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 546.7725ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:07.715349311Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 715.305773ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:08.028174622Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:08.248343683Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"820.405938ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 712.023417ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:08.272486209Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 975.185276ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:08.312302616Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 663.809571ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:08.576765846Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 1.21030963s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:08.650354799Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 1.242565687s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:08.753428691Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 1.06121985s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:08.997919159Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 579.694857ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:09.253604578Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 641.230388ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:09.300355489Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 1.135381651s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:09.845926903Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 1.265749349s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:09.875037701Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 1.159473841s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:09.955302258Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 945.728689ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:10.011058126Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 1.442830229s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:10.073551615Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:10.466164063Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62032"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:10.466271126Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"2c3e2d0a-14f8-409c-9532-51bd9f1eac1a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62032","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 1.252888229s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:10.965739628Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 655.393171ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:10.99103498Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 1.209334383s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:11.122510202Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 1.353265262s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:11.149638132Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 1.184563641s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:11.524364441Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 609.452867ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:11.654414864Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 530.820728ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:12.228082757Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 916.014656ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:12.271980982Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba\") failed: Property not found. retry in 1.375936526s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:12.340177799Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c\") failed: Property not found. retry in 1.20567022s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:12.51232829Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf\") failed: Property not found. retry in 979.798377ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:12.718077054Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7\") failed: Property not found. retry in 1.151425096s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:12.767108875Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7\") failed: Property not found. retry in 1.279627159s","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:13.197919331Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:14.498701683Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:14.498761374Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_cockroachdb_5c749b5a-3b6c-44a8-9cb5-f97707372cba"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:14.498768424Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_cockroachdb_3cc04303-718d-447d-bc89-3a4154b619d7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:14.498776614Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_cockroachdb_a52f73c1-a1a5-4560-812f-18319626624c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:14.498784624Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_cockroachdb_57ed5e38-59cc-4e8b-8338-2dceef12fef7"} {"msg":"Writing ledger to /pool/int/484faba5-5bf0-4a1e-b697-76cf9fed3ca8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:14.498812675Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/88c34544-1e3c-4ff1-a021-704e84e5eabd/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:14.499205603Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:14.499583641Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"41b451dd-f63b-476f-bd83-ac34a01d843e","remote_addr":"[fd00:1122:3344:101::1]:64045","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":21947362,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:14.533747756Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:51607"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:14.560401166Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"eb19acf1-f036-4108-a5b0-d37de03e7e3c","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":26491,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:14.56063107Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"88.170036ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eb19acf1-f036-4108-a5b0-d37de03e7e3c\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:39:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eb19acf1-f036-4108-a5b0-d37de03e7e3c\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:14.676080472Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"c21c27ee-ad3f-4925-ac5e-b0ea9abdf6ef","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":26162,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:14.676245885Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"141.967125ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c21c27ee-ad3f-4925-ac5e-b0ea9abdf6ef\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:39:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c21c27ee-ad3f-4925-ac5e-b0ea9abdf6ef\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:14.846604499Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"23dcef54-e5bc-4948-a86e-a95cd0dcbc18","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":27163,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:14.846773873Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"207.374106ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"23dcef54-e5bc-4948-a86e-a95cd0dcbc18\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:39:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"23dcef54-e5bc-4948-a86e-a95cd0dcbc18\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:15.088159672Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:35.396928172Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:38.964646886Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"03cd9689-0398-4e92-882c-4c12b6f02068","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":23909089,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:38.971024687Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"132.725747ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"03cd9689-0398-4e92-882c-4c12b6f02068\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:39:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"03cd9689-0398-4e92-882c-4c12b6f02068\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:38.988855965Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"2.98968073s"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:39.143710119Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:39.255493795Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:44.340356183Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"a8175414-f6bf-4476-9fc9-698aafd2aba1","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":5235314,"response_code":"500"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:44.341209051Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56565"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:44.341347874Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"189.508529ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a8175414-f6bf-4476-9fc9-698aafd2aba1\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:39:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a8175414-f6bf-4476-9fc9-698aafd2aba1\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:44.342441676Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"dde95b1d-132c-46c0-b8fa-84e2b6656d0a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56565","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":525,"response_code":"200"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:44.560444433Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:44.676405155Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:50.031731052Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"54a7474e-9d13-4004-ab6c-5fa8fc5d345c","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":5497953,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:39:50.032301334Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"144.726207ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"54a7474e-9d13-4004-ab6c-5fa8fc5d345c\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:39:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"54a7474e-9d13-4004-ab6c-5fa8fc5d345c\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:50.206892215Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:39:50.321412347Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:03.435287948Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"b36e06ae-59db-45eb-9d1b-594c73d81c6f","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":13257329,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:40:03.435957921Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"446.74437ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b36e06ae-59db-45eb-9d1b-594c73d81c6f\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:40:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b36e06ae-59db-45eb-9d1b-594c73d81c6f\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:40:03.439442484Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"4.511325458s"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:03.911561782Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:04.024092163Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:12.373426665Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"2f6945aa-f538-4dc3-8d55-bcee2b283c01","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":8489711,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:40:12.374014317Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"484.485098ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2f6945aa-f538-4dc3-8d55-bcee2b283c01\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:40:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2f6945aa-f538-4dc3-8d55-bcee2b283c01\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:12.8890043Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:13.008718229Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:19.838388095Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"6b3d6ee2-615d-4903-983d-f8d4e814b182","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":6977897,"response_code":"500"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:19.838880795Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64367"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:40:19.83910034Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"457.030831ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6b3d6ee2-615d-4903-983d-f8d4e814b182\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:40:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6b3d6ee2-615d-4903-983d-f8d4e814b182\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:19.839655861Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"7714bf71-24c6-4d27-a7ff-1ceba4177d62","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64367","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":68,"response_code":"200"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:20.326753988Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:20.439410862Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:27.41707602Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"6ff41acc-9fc6-4b9d-a095-c1666a1d944a","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":7119595,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:40:27.41752928Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"471.830844ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6ff41acc-9fc6-4b9d-a095-c1666a1d944a\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:40:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6ff41acc-9fc6-4b9d-a095-c1666a1d944a\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:40:27.420309797Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"6.243760831s"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:27.917936482Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:28.03178916Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:37.671146902Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"e7331efc-e739-4d11-8fe7-b7164e78578c","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":9780972,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:40:37.671725814Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"311.272887ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e7331efc-e739-4d11-8fe7-b7164e78578c\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:40:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e7331efc-e739-4d11-8fe7-b7164e78578c\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:38.012510103Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:38.121790697Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:46.75863728Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"c9833179-2a4f-4e3f-b4f6-1f57c67aa4a5","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":8774657,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:40:46.75913493Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"683.237759ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c9833179-2a4f-4e3f-b4f6-1f57c67aa4a5\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:40:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c9833179-2a4f-4e3f-b4f6-1f57c67aa4a5\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:47.469639006Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:40:47.585068607Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:05.069825605Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"a252a69d-54fb-48b8-82a6-dc53cdc1645b","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":17627187,"response_code":"500"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:05.070109821Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50224"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:41:05.070714663Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"1.050340561s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a252a69d-54fb-48b8-82a6-dc53cdc1645b\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:41:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a252a69d-54fb-48b8-82a6-dc53cdc1645b\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:41:05.07395735Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"17.25744959s"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:06.149583777Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:07.307210566Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:15.487707745Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"a2579291-416d-426c-8e00-752edc559605","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":9366119,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:41:15.488214225Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"1.459322889s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a2579291-416d-426c-8e00-752edc559605\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:41:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a2579291-416d-426c-8e00-752edc559605\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:16.976657608Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:17.085339379Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:32.250179775Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"47415430-492f-4f61-a0c0-578376e972ca","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":15301104,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:41:32.25090363Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"678.857632ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"47415430-492f-4f61-a0c0-578376e972ca\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:41:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"47415430-492f-4f61-a0c0-578376e972ca\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:32.960538828Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:33.090490538Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:40.585442297Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"d7ec36b6-84c5-4a2a-8e0e-5d1d8ecae5af","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":7654835,"response_code":"500"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:40.585808124Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55055"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:41:40.585991008Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"909.606614ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d7ec36b6-84c5-4a2a-8e0e-5d1d8ecae5af\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:41:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d7ec36b6-84c5-4a2a-8e0e-5d1d8ecae5af\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:40.58609664Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"9197dae0-3f48-4b40-895a-56684a6c05d6","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55055","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:41.52658221Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:41:41.643402839Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:05.463600909Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"a5794071-cc7b-4624-ba24-2c42f4315b5d","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":23966254,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:42:05.464389476Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"1.598898035s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a5794071-cc7b-4624-ba24-2c42f4315b5d\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:42:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a5794071-cc7b-4624-ba24-2c42f4315b5d\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:42:05.468812077Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"38.625099715s"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:07.092355065Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:07.204196443Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:15.676719446Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"3e77e2d6-cf48-4d7c-af80-2c51bb614833","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":8612363,"response_code":"500"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:15.677230336Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36300"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:42:15.677378999Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"2.044919738s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3e77e2d6-cf48-4d7c-af80-2c51bb614833\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:42:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3e77e2d6-cf48-4d7c-af80-2c51bb614833\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:15.677507092Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"e7d2f2b3-60e5-480b-a388-12123cc0e17f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36300","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102,"response_code":"200"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:17.751625885Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:17.86627948Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:25.048713322Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"16a8e5f7-1373-422e-8bce-59c9298433c6","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":7325053,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:42:25.049124871Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"2.159785639s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"16a8e5f7-1373-422e-8bce-59c9298433c6\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:42:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"16a8e5f7-1373-422e-8bce-59c9298433c6\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:27.237202654Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:27.348555681Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:34.495680415Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"60ae877a-f862-4e0c-bc25-42cb280673af","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":7286049,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:42:34.496065113Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"4.219573675s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"60ae877a-f862-4e0c-bc25-42cb280673af\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:42:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"60ae877a-f862-4e0c-bc25-42cb280673af\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:38.74470633Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:38.857074477Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:46.319988015Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"78b8cdd5-4c84-45da-a39e-14241e42e368","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":7602942,"response_code":"500"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:46.320345813Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64561"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:42:46.32072545Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"5.19925374s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"78b8cdd5-4c84-45da-a39e-14241e42e368\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:42:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"78b8cdd5-4c84-45da-a39e-14241e42e368\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:46.322757522Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"d786d620-a19b-4e21-8881-c9994a35a99a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64561","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95,"response_code":"200"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:51.547950223Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:42:51.658950312Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:16.092720288Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"bcf7fc03-452d-4ab4-97b6-f00dd29c6f0f","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":24572100,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:43:16.093204468Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"2.406606897s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bcf7fc03-452d-4ab4-97b6-f00dd29c6f0f\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:43:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bcf7fc03-452d-4ab4-97b6-f00dd29c6f0f\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:16.094170608Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:43:16.096095678Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e358e662-7828-4f7f-85a4-21e089483ee6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"59.283718609s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:16.355984228Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47701"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:16.356090961Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"88231df0-a07f-4db0-a0a6-6587d0694545","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47701","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"200"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:18.527992001Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:18.640663885Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:25.749732324Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"03a00770-2a3a-4098-85a6-29f10216bb04","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":7249185,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:43:25.750124492Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"5.535295984s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"03a00770-2a3a-4098-85a6-29f10216bb04\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:43:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"03a00770-2a3a-4098-85a6-29f10216bb04\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:31.313194081Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:31.420829231Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:48.707037365Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"285e3053-38a8-4559-821e-e411f4598815","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":17420363,"response_code":"500"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:48.707366502Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45801"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:43:48.707661558Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"3.527113508s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"285e3053-38a8-4559-821e-e411f4598815\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:43:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"285e3053-38a8-4559-821e-e411f4598815\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:48.707675318Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"4958953c-666d-4b7c-a489-24f0d43b4d63","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45801","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"200"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:52.266200109Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:52.383021489Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-19T00:43:59.97151316Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"3a6c48d7-6611-4013-8491-142f005c938a","remote_addr":"[fd00:1122:3344:101::1]:51607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_05bfdf27-226d-408e-8634-4cf7d9392bcf': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nINSERT 1\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCOMMIT\nBEGIN\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE SEQUENCE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nINSERT 1\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nERROR: at or near \")\": syntax error\nSQLSTATE: 42601\nDETAIL: source SQL:\nCREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (\n -- UUID of this dispatch.\n id UUID PRIMARY KEY,\n -- UUID of the webhook receiver (foreign key into\n -- `omicron.public.webhook_rx`)\n rx_id UUID NOT NULL,\n payload JSONB NOT NULL,\n time_created TIMESTAMPTZ NOT NULL,\n -- If this is set, then this webhook message has either been delivered\n -- successfully, or is considered permanently failed.\n time_completed TIMESTAMPTZ,\n)\n^\nHINT: try \\h CREATE TABLE\nFailed running \"sql\"\n","latency_us":7385641,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-19T00:43:59.971935389Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"7.394290669s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3a6c48d7-6611-4013-8491-142f005c938a\", \"content-length\": \"124\", \"date\": \"Thu, 19 Dec 2024 00:43:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3a6c48d7-6611-4013-8491-142f005c938a\" }"}